1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell. 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 12 #include "rvu_struct.h" 13 #include "rvu_reg.h" 14 #include "rvu.h" 15 #include "npc.h" 16 #include "cgx.h" 17 #include "npc_profile.h" 18 #include "rvu_npc_hash.h" 19 #include "cn20k/npc.h" 20 #include "rvu_npc.h" 21 #include "cn20k/reg.h" 22 23 #define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */ 24 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 25 26 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 27 #define NPC_HW_TSTAMP_OFFSET 8ULL 28 #define NPC_KEX_CHAN_MASK 0xFFFULL 29 #define NPC_KEX_PF_FUNC_MASK 0xFFFFULL 30 31 #define ALIGN_8B_CEIL(__a) (((__a) + 7) & (-8)) 32 33 static const char def_pfl_name[] = "default"; 34 35 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 36 int blkaddr, u16 pcifunc); 37 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 38 u16 pcifunc); 39 40 bool is_npc_intf_tx(u8 intf) 41 { 42 return !!(intf & 0x1); 43 } 44 45 bool is_npc_intf_rx(u8 intf) 46 { 47 return !(intf & 0x1); 48 } 49 50 bool is_npc_interface_valid(struct rvu *rvu, u8 intf) 51 { 52 struct rvu_hwinfo *hw = rvu->hw; 53 54 return intf < hw->npc_intfs; 55 } 56 57 int rvu_npc_get_tx_nibble_cfg(struct rvu *rvu, u64 nibble_ena) 58 { 59 /* Due to a HW issue in these silicon versions, parse nibble enable 60 * configuration has to be identical for both Rx and Tx interfaces. 61 */ 62 if (is_rvu_96xx_B0(rvu)) 63 return nibble_ena; 64 return 0; 65 } 66 67 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 68 { 69 int blkaddr; 70 u64 val = 0; 71 72 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 73 if (blkaddr < 0) 74 return; 75 76 /* Config CPI base for the PKIND */ 77 val = pkind | 1ULL << 62; 78 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 79 } 80 81 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 82 { 83 struct npc_pkind *pkind = &rvu->hw->pkind; 84 u32 map; 85 int i; 86 87 for (i = 0; i < pkind->rsrc.max; i++) { 88 map = pkind->pfchan_map[i]; 89 if (((map >> 16) & 0x3F) == pf) 90 return i; 91 } 92 return -1; 93 } 94 95 #define NPC_AF_ACTION0_PTR_ADVANCE GENMASK_ULL(27, 20) 96 97 int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable) 98 { 99 int pkind, blkaddr; 100 u64 val; 101 102 pkind = rvu_npc_get_pkind(rvu, pf); 103 if (pkind < 0) { 104 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__); 105 return -EINVAL; 106 } 107 108 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 109 if (blkaddr < 0) { 110 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 111 return -EINVAL; 112 } 113 114 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 115 val &= ~NPC_AF_ACTION0_PTR_ADVANCE; 116 /* If timestamp is enabled then configure NPC to shift 8 bytes */ 117 if (enable) 118 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE, 119 NPC_HW_TSTAMP_OFFSET); 120 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 121 122 return 0; 123 } 124 125 static int npc_get_ucast_mcam_index(struct npc_mcam *mcam, u16 pcifunc, 126 int nixlf) 127 { 128 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 129 struct rvu *rvu = hw->rvu; 130 int blkaddr = 0, max = 0; 131 struct rvu_block *block; 132 struct rvu_pfvf *pfvf; 133 134 pfvf = rvu_get_pfvf(rvu, pcifunc); 135 /* Given a PF/VF and NIX LF number calculate the unicast mcam 136 * entry index based on the NIX block assigned to the PF/VF. 137 */ 138 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 139 while (blkaddr) { 140 if (pfvf->nix_blkaddr == blkaddr) 141 break; 142 block = &rvu->hw->block[blkaddr]; 143 max += block->lf.max; 144 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 145 } 146 147 return mcam->nixlf_offset + (max + nixlf) * RSVD_MCAM_ENTRIES_PER_NIXLF; 148 } 149 150 int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 151 u16 pcifunc, int nixlf, int type) 152 { 153 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 154 struct rvu *rvu = hw->rvu; 155 u16 bcast, mcast, promisc, ucast; 156 int index; 157 int rc; 158 int pf; 159 160 if (is_cn20k(rvu->pdev)) { 161 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, 162 &promisc, &ucast); 163 if (rc) 164 return -EFAULT; 165 166 switch (type) { 167 case NIXLF_BCAST_ENTRY: 168 return bcast; 169 case NIXLF_ALLMULTI_ENTRY: 170 return mcast; 171 case NIXLF_PROMISC_ENTRY: 172 return promisc; 173 case NIXLF_UCAST_ENTRY: 174 return ucast; 175 default: 176 return -EINVAL; 177 } 178 } 179 180 /* Check if this is for a PF */ 181 pf = rvu_get_pf(rvu->pdev, pcifunc); 182 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 183 /* Reserved entries exclude PF0 */ 184 pf--; 185 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 186 /* Broadcast address matching entry should be first so 187 * that the packet can be replicated to all VFs. 188 */ 189 if (type == NIXLF_BCAST_ENTRY) 190 return index; 191 else if (type == NIXLF_ALLMULTI_ENTRY) 192 return index + 1; 193 else if (type == NIXLF_PROMISC_ENTRY) 194 return index + 2; 195 } 196 197 return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf); 198 } 199 200 int npc_get_bank(struct npc_mcam *mcam, int index) 201 { 202 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 203 int bank = index / mcam->banksize; 204 struct rvu *rvu = hw->rvu; 205 206 if (is_cn20k(rvu->pdev)) 207 return bank; 208 209 /* 0,1 & 2,3 banks are combined for this keysize */ 210 if (mcam->keysize == NPC_MCAM_KEY_X2) 211 return bank ? 2 : 0; 212 213 return bank; 214 } 215 216 bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 217 int blkaddr, int index) 218 { 219 int bank = npc_get_bank(mcam, index); 220 u64 cfg; 221 222 index &= (mcam->banksize - 1); 223 if (is_cn20k(rvu->pdev)) 224 cfg = rvu_read64(rvu, blkaddr, 225 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, 226 bank)); 227 else 228 cfg = rvu_read64(rvu, blkaddr, 229 NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 230 231 return (cfg & 1); 232 } 233 234 void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 235 int blkaddr, int index, bool enable) 236 { 237 int bank = npc_get_bank(mcam, index); 238 int actbank = bank; 239 240 if (is_cn20k(rvu->pdev)) { 241 if (index < 0 || index >= mcam->banksize * mcam->banks) 242 return; 243 244 return npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable); 245 } 246 247 index &= (mcam->banksize - 1); 248 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 249 rvu_write64(rvu, blkaddr, 250 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 251 enable ? 1 : 0); 252 } 253 } 254 255 static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 256 int blkaddr, int index) 257 { 258 int bank = npc_get_bank(mcam, index); 259 int actbank = bank; 260 261 index &= (mcam->banksize - 1); 262 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 263 rvu_write64(rvu, blkaddr, 264 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0); 265 rvu_write64(rvu, blkaddr, 266 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0); 267 268 rvu_write64(rvu, blkaddr, 269 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0); 270 rvu_write64(rvu, blkaddr, 271 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0); 272 273 rvu_write64(rvu, blkaddr, 274 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0); 275 rvu_write64(rvu, blkaddr, 276 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0); 277 } 278 } 279 280 static void npc_get_keyword(struct mcam_entry *entry, int idx, 281 u64 *cam0, u64 *cam1) 282 { 283 u64 kw_mask = 0x00; 284 285 #define CAM_MASK(n) (BIT_ULL(n) - 1) 286 287 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 288 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 289 * 290 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 291 */ 292 switch (idx) { 293 case 0: 294 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 295 *cam1 = entry->kw[0]; 296 kw_mask = entry->kw_mask[0]; 297 break; 298 case 1: 299 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 300 *cam1 = entry->kw[1] & CAM_MASK(48); 301 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 302 break; 303 case 2: 304 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 305 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 306 */ 307 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 308 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 309 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 310 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 311 break; 312 case 3: 313 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 314 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 315 */ 316 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 317 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 318 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 319 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 320 break; 321 case 4: 322 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 323 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 324 */ 325 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 326 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 327 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 328 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 329 break; 330 case 5: 331 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 332 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 333 */ 334 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 335 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 336 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 337 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 338 break; 339 case 6: 340 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 341 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 342 */ 343 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 344 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 345 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 346 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 347 break; 348 case 7: 349 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 350 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 351 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 352 break; 353 } 354 355 *cam1 &= kw_mask; 356 *cam0 = ~*cam1 & kw_mask; 357 } 358 359 static void npc_fill_entryword(struct mcam_entry *entry, int idx, 360 u64 cam0, u64 cam1) 361 { 362 /* Similar to npc_get_keyword, but fills mcam_entry structure from 363 * CAM registers. 364 */ 365 switch (idx) { 366 case 0: 367 entry->kw[0] = cam1; 368 entry->kw_mask[0] = cam1 ^ cam0; 369 break; 370 case 1: 371 entry->kw[1] = cam1; 372 entry->kw_mask[1] = cam1 ^ cam0; 373 break; 374 case 2: 375 entry->kw[1] |= (cam1 & CAM_MASK(16)) << 48; 376 entry->kw[2] = (cam1 >> 16) & CAM_MASK(48); 377 entry->kw_mask[1] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 378 entry->kw_mask[2] = ((cam1 ^ cam0) >> 16) & CAM_MASK(48); 379 break; 380 case 3: 381 entry->kw[2] |= (cam1 & CAM_MASK(16)) << 48; 382 entry->kw[3] = (cam1 >> 16) & CAM_MASK(32); 383 entry->kw_mask[2] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 384 entry->kw_mask[3] = ((cam1 ^ cam0) >> 16) & CAM_MASK(32); 385 break; 386 case 4: 387 entry->kw[3] |= (cam1 & CAM_MASK(32)) << 32; 388 entry->kw[4] = (cam1 >> 32) & CAM_MASK(32); 389 entry->kw_mask[3] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 390 entry->kw_mask[4] = ((cam1 ^ cam0) >> 32) & CAM_MASK(32); 391 break; 392 case 5: 393 entry->kw[4] |= (cam1 & CAM_MASK(32)) << 32; 394 entry->kw[5] = (cam1 >> 32) & CAM_MASK(16); 395 entry->kw_mask[4] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 396 entry->kw_mask[5] = ((cam1 ^ cam0) >> 32) & CAM_MASK(16); 397 break; 398 case 6: 399 entry->kw[5] |= (cam1 & CAM_MASK(48)) << 16; 400 entry->kw[6] = (cam1 >> 48) & CAM_MASK(16); 401 entry->kw_mask[5] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 402 entry->kw_mask[6] = ((cam1 ^ cam0) >> 48) & CAM_MASK(16); 403 break; 404 case 7: 405 entry->kw[6] |= (cam1 & CAM_MASK(48)) << 16; 406 entry->kw_mask[6] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 407 break; 408 } 409 } 410 411 static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, 412 int blkaddr, u16 pf_func) 413 { 414 int bank, nixlf, index; 415 u64 reg; 416 417 /* get ucast entry rule entry index */ 418 if (nix_get_nixlf(rvu, pf_func, &nixlf, NULL)) { 419 dev_err(rvu->dev, "%s: nixlf not attached to pcifunc:0x%x\n", 420 __func__, pf_func); 421 /* Action 0 is drop */ 422 return 0; 423 } 424 425 index = npc_get_nixlf_mcam_index(mcam, pf_func, nixlf, 426 NIXLF_UCAST_ENTRY); 427 bank = npc_get_bank(mcam, index); 428 index &= (mcam->banksize - 1); 429 430 if (is_cn20k(rvu->pdev)) 431 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); 432 else 433 reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); 434 435 return rvu_read64(rvu, blkaddr, reg); 436 } 437 438 static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, 439 int blkaddr, int index, struct mcam_entry *entry, 440 bool *enable) 441 { 442 struct rvu_npc_mcam_rule *rule; 443 u16 owner, target_func; 444 struct rvu_pfvf *pfvf; 445 u64 rx_action; 446 447 owner = mcam->entry2pfvf_map[index]; 448 target_func = (entry->action >> 4) & 0xffff; 449 /* do nothing when target is LBK/PF or owner is not PF */ 450 if (is_pffunc_af(owner) || is_lbk_vf(rvu, target_func) || 451 (owner & RVU_PFVF_FUNC_MASK) || 452 !(target_func & RVU_PFVF_FUNC_MASK)) 453 return; 454 455 /* save entry2target_pffunc */ 456 pfvf = rvu_get_pfvf(rvu, target_func); 457 mcam->entry2target_pffunc[index] = target_func; 458 459 /* don't enable rule when nixlf not attached or initialized */ 460 if (!(is_nixlf_attached(rvu, target_func) && 461 test_bit(NIXLF_INITIALIZED, &pfvf->flags))) 462 *enable = false; 463 464 /* fix up not needed for the rules added by user(ntuple filters) */ 465 list_for_each_entry(rule, &mcam->mcam_rules, list) { 466 if (rule->entry == index) 467 return; 468 } 469 470 /* AF modifies given action iff PF/VF has requested for it */ 471 if ((entry->action & 0xFULL) != NIX_RX_ACTION_DEFAULT) 472 return; 473 474 /* copy VF default entry action to the VF mcam entry */ 475 rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr, 476 target_func); 477 if (rx_action) 478 entry->action = rx_action; 479 } 480 481 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 482 int blkaddr, int index, u8 intf, 483 struct mcam_entry *entry, bool enable) 484 { 485 int bank = npc_get_bank(mcam, index); 486 int kw = 0, actbank, actindex; 487 u8 tx_intf_mask = ~intf & 0x3; 488 u8 tx_intf = intf; 489 u64 cam0, cam1; 490 491 actbank = bank; /* Save bank id, to set action later on */ 492 actindex = index; 493 index &= (mcam->banksize - 1); 494 495 /* Disable before mcam entry update */ 496 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 497 498 /* Clear mcam entry to avoid writes being suppressed by NPC */ 499 npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex); 500 501 /* CAM1 takes the comparison value and 502 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 503 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 504 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 505 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 506 */ 507 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 508 /* Interface should be set in all banks */ 509 if (is_npc_intf_tx(intf)) { 510 /* Last bit must be set and rest don't care 511 * for TX interfaces 512 */ 513 tx_intf_mask = 0x1; 514 tx_intf = intf & tx_intf_mask; 515 tx_intf_mask = ~tx_intf & tx_intf_mask; 516 } 517 518 rvu_write64(rvu, blkaddr, 519 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 520 tx_intf); 521 rvu_write64(rvu, blkaddr, 522 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 523 tx_intf_mask); 524 525 /* Set the match key */ 526 npc_get_keyword(entry, kw, &cam0, &cam1); 527 rvu_write64(rvu, blkaddr, 528 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 529 rvu_write64(rvu, blkaddr, 530 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 531 532 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 533 rvu_write64(rvu, blkaddr, 534 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 535 rvu_write64(rvu, blkaddr, 536 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 537 } 538 539 /* PF installing VF rule */ 540 if (is_npc_intf_rx(intf) && actindex < mcam->bmap_entries) 541 npc_fixup_vf_rule(rvu, mcam, blkaddr, actindex, entry, &enable); 542 543 /* Set 'action' */ 544 rvu_write64(rvu, blkaddr, 545 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 546 547 /* Set TAG 'action' */ 548 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 549 entry->vtag_action); 550 551 /* Enable the entry */ 552 if (enable) 553 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 554 } 555 556 void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 557 int blkaddr, u16 src, 558 struct mcam_entry *entry, u8 *intf, u8 *ena) 559 { 560 int sbank = npc_get_bank(mcam, src); 561 int bank, kw = 0; 562 u64 cam0, cam1; 563 564 src &= (mcam->banksize - 1); 565 bank = sbank; 566 567 for (; bank < (sbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 568 cam1 = rvu_read64(rvu, blkaddr, 569 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 1)); 570 cam0 = rvu_read64(rvu, blkaddr, 571 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 0)); 572 npc_fill_entryword(entry, kw, cam0, cam1); 573 574 cam1 = rvu_read64(rvu, blkaddr, 575 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 1)); 576 cam0 = rvu_read64(rvu, blkaddr, 577 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 0)); 578 npc_fill_entryword(entry, kw + 1, cam0, cam1); 579 } 580 581 entry->action = rvu_read64(rvu, blkaddr, 582 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 583 entry->vtag_action = 584 rvu_read64(rvu, blkaddr, 585 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 586 *intf = rvu_read64(rvu, blkaddr, 587 NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank, 1)) & 3; 588 *ena = rvu_read64(rvu, blkaddr, 589 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1; 590 } 591 592 static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 593 int blkaddr, u16 src, u16 dest) 594 { 595 int dbank = npc_get_bank(mcam, dest); 596 int sbank = npc_get_bank(mcam, src); 597 u64 cfg, sreg, dreg; 598 int bank, i; 599 600 if (is_cn20k(rvu->pdev)) 601 return npc_cn20k_copy_mcam_entry(rvu, blkaddr, src, dest); 602 603 src &= (mcam->banksize - 1); 604 dest &= (mcam->banksize - 1); 605 606 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 607 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 608 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 609 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 610 for (i = 0; i < 6; i++) { 611 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 612 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 613 } 614 } 615 616 /* Copy action */ 617 cfg = rvu_read64(rvu, blkaddr, 618 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 619 rvu_write64(rvu, blkaddr, 620 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 621 622 /* Copy TAG action */ 623 cfg = rvu_read64(rvu, blkaddr, 624 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 625 rvu_write64(rvu, blkaddr, 626 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 627 628 /* Enable or disable */ 629 cfg = rvu_read64(rvu, blkaddr, 630 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 631 rvu_write64(rvu, blkaddr, 632 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 633 } 634 635 u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 636 int blkaddr, int index) 637 { 638 int bank = npc_get_bank(mcam, index); 639 u64 reg; 640 641 index &= (mcam->banksize - 1); 642 643 if (is_cn20k(rvu->pdev)) 644 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); 645 else 646 reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); 647 return rvu_read64(rvu, blkaddr, reg); 648 } 649 650 void npc_set_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 651 int blkaddr, int index, u64 cfg) 652 { 653 int bank = npc_get_bank(mcam, index); 654 u64 reg; 655 656 index &= (mcam->banksize - 1); 657 658 if (is_cn20k(rvu->pdev)) 659 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); 660 else 661 reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); 662 663 return rvu_write64(rvu, blkaddr, reg, cfg); 664 } 665 666 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 667 int nixlf, u64 chan, u8 *mac_addr) 668 { 669 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 670 struct npc_install_flow_req req = { 0 }; 671 struct npc_install_flow_rsp rsp = { 0 }; 672 struct npc_mcam *mcam = &rvu->hw->mcam; 673 struct nix_rx_action action = { 0 }; 674 int blkaddr, index; 675 676 /* AF's and SDP VFs work in promiscuous mode */ 677 if (is_lbk_vf(rvu, pcifunc) || is_sdp_vf(rvu, pcifunc)) 678 return; 679 680 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 681 if (blkaddr < 0) 682 return; 683 684 /* Ucast rule should not be installed if DMAC 685 * extraction is not supported by the profile. 686 */ 687 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf)) 688 return; 689 690 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 691 nixlf, NIXLF_UCAST_ENTRY); 692 693 /* Don't change the action if entry is already enabled 694 * Otherwise RSS action may get overwritten. 695 */ 696 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 697 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 698 blkaddr, index); 699 } else { 700 action.op = NIX_RX_ACTIONOP_UCAST; 701 action.pf_func = pcifunc; 702 } 703 704 req.default_rule = 1; 705 ether_addr_copy(req.packet.dmac, mac_addr); 706 eth_broadcast_addr((u8 *)&req.mask.dmac); 707 req.features = BIT_ULL(NPC_DMAC); 708 req.channel = chan; 709 req.chan_mask = 0xFFFU; 710 req.intf = pfvf->nix_rx_intf; 711 req.op = action.op; 712 req.hdr.pcifunc = 0; /* AF is requester */ 713 req.vf = action.pf_func; 714 req.index = action.index; 715 req.match_id = action.match_id; 716 req.flow_key_alg = action.flow_key_alg; 717 718 if (is_cn20k(rvu->pdev)) 719 req.hw_prio = pfvf->hw_prio; 720 721 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 722 } 723 724 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 725 int nixlf, u64 chan, u8 chan_cnt) 726 { 727 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 728 struct npc_install_flow_req req = { 0 }; 729 struct npc_install_flow_rsp rsp = { 0 }; 730 struct npc_mcam *mcam = &rvu->hw->mcam; 731 struct rvu_hwinfo *hw = rvu->hw; 732 int blkaddr, ucast_idx, index; 733 struct nix_rx_action action = { 0 }; 734 u64 relaxed_mask; 735 u8 flow_key_alg; 736 737 if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc)) 738 return; 739 740 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 741 if (blkaddr < 0) 742 return; 743 744 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 745 nixlf, NIXLF_PROMISC_ENTRY); 746 747 if (is_cgx_vf(rvu, pcifunc)) 748 index = npc_get_nixlf_mcam_index(mcam, 749 pcifunc & ~RVU_PFVF_FUNC_MASK, 750 nixlf, NIXLF_PROMISC_ENTRY); 751 752 /* If the corresponding PF's ucast action is RSS, 753 * use the same action for promisc also 754 */ 755 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 756 nixlf, NIXLF_UCAST_ENTRY); 757 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 758 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 759 blkaddr, ucast_idx); 760 761 if (action.op != NIX_RX_ACTIONOP_RSS) { 762 *(u64 *)&action = 0; 763 action.op = NIX_RX_ACTIONOP_UCAST; 764 } 765 766 flow_key_alg = action.flow_key_alg; 767 768 /* RX_ACTION set to MCAST for CGX PF's */ 769 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list && 770 is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc))) { 771 *(u64 *)&action = 0; 772 action.op = NIX_RX_ACTIONOP_MCAST; 773 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 774 action.index = pfvf->promisc_mce_idx; 775 } 776 777 /* For cn10k the upper two bits of the channel number are 778 * cpt channel number. with masking out these bits in the 779 * mcam entry, same entry used for NIX will allow packets 780 * received from cpt for parsing. 781 */ 782 if (!is_rvu_otx2(rvu)) { 783 req.chan_mask = NIX_CHAN_CPT_X2P_MASK; 784 } else { 785 req.chan_mask = 0xFFFU; 786 } 787 788 if (chan_cnt > 1) { 789 if (!is_power_of_2(chan_cnt)) { 790 dev_err(rvu->dev, 791 "%s: channel count more than 1, must be power of 2\n", __func__); 792 return; 793 } 794 relaxed_mask = GENMASK_ULL(BITS_PER_LONG_LONG - 1, 795 ilog2(chan_cnt)); 796 req.chan_mask &= relaxed_mask; 797 } 798 799 req.channel = chan; 800 req.intf = pfvf->nix_rx_intf; 801 req.entry = index; 802 req.op = action.op; 803 req.hdr.pcifunc = 0; /* AF is requester */ 804 req.vf = pcifunc; 805 req.index = action.index; 806 req.match_id = action.match_id; 807 req.flow_key_alg = flow_key_alg; 808 809 if (is_cn20k(rvu->pdev)) 810 req.hw_prio = pfvf->hw_prio; 811 812 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 813 } 814 815 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, 816 int nixlf, bool enable) 817 { 818 struct npc_mcam *mcam = &rvu->hw->mcam; 819 int blkaddr, index; 820 821 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 822 if (blkaddr < 0) 823 return; 824 825 /* Get 'pcifunc' of PF device */ 826 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 827 828 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 829 nixlf, NIXLF_PROMISC_ENTRY); 830 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 831 } 832 833 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 834 int nixlf, u64 chan) 835 { 836 struct rvu_pfvf *pfvf; 837 struct npc_install_flow_req req = { 0 }; 838 struct npc_install_flow_rsp rsp = { 0 }; 839 struct npc_mcam *mcam = &rvu->hw->mcam; 840 struct rvu_hwinfo *hw = rvu->hw; 841 int blkaddr, index; 842 843 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 844 if (blkaddr < 0) 845 return; 846 847 /* Skip LBK VFs */ 848 if (is_lbk_vf(rvu, pcifunc)) 849 return; 850 851 /* If pkt replication is not supported, 852 * then only PF is allowed to add a bcast match entry. 853 */ 854 if (!hw->cap.nix_rx_multicast && is_vf(pcifunc)) 855 return; 856 857 /* Get 'pcifunc' of PF device */ 858 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 859 pfvf = rvu_get_pfvf(rvu, pcifunc); 860 861 /* Bcast rule should not be installed if both DMAC 862 * and LXMB extraction is not supported by the profile. 863 */ 864 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf) && 865 !npc_is_feature_supported(rvu, BIT_ULL(NPC_LXMB), pfvf->nix_rx_intf)) 866 return; 867 868 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 869 nixlf, NIXLF_BCAST_ENTRY); 870 871 if (!hw->cap.nix_rx_multicast) { 872 /* Early silicon doesn't support pkt replication, 873 * so install entry with UCAST action, so that PF 874 * receives all broadcast packets. 875 */ 876 req.op = NIX_RX_ACTIONOP_UCAST; 877 } else { 878 req.op = NIX_RX_ACTIONOP_MCAST; 879 req.index = pfvf->bcast_mce_idx; 880 } 881 882 eth_broadcast_addr((u8 *)&req.packet.dmac); 883 eth_broadcast_addr((u8 *)&req.mask.dmac); 884 req.features = BIT_ULL(NPC_DMAC); 885 req.channel = chan; 886 req.chan_mask = 0xFFFU; 887 req.intf = pfvf->nix_rx_intf; 888 req.entry = index; 889 req.hdr.pcifunc = 0; /* AF is requester */ 890 req.vf = pcifunc; 891 892 if (is_cn20k(rvu->pdev)) 893 req.hw_prio = pfvf->hw_prio; 894 895 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 896 } 897 898 void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 899 u64 chan) 900 { 901 struct npc_install_flow_req req = { 0 }; 902 struct npc_install_flow_rsp rsp = { 0 }; 903 struct npc_mcam *mcam = &rvu->hw->mcam; 904 struct rvu_hwinfo *hw = rvu->hw; 905 int blkaddr, ucast_idx, index; 906 u8 mac_addr[ETH_ALEN] = { 0 }; 907 struct nix_rx_action action = { 0 }; 908 struct rvu_pfvf *pfvf; 909 u8 flow_key_alg; 910 u16 vf_func; 911 912 /* Only CGX PF/VF can add allmulticast entry */ 913 if (is_lbk_vf(rvu, pcifunc) && is_sdp_vf(rvu, pcifunc)) 914 return; 915 916 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 917 if (blkaddr < 0) 918 return; 919 920 /* Get 'pcifunc' of PF device */ 921 vf_func = pcifunc & RVU_PFVF_FUNC_MASK; 922 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 923 pfvf = rvu_get_pfvf(rvu, pcifunc); 924 925 /* Mcast rule should not be installed if both DMAC 926 * and LXMB extraction is not supported by the profile. 927 */ 928 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf) && 929 !npc_is_feature_supported(rvu, BIT_ULL(NPC_LXMB), pfvf->nix_rx_intf)) 930 return; 931 932 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 933 nixlf, NIXLF_ALLMULTI_ENTRY); 934 935 /* If the corresponding PF's ucast action is RSS, 936 * use the same action for multicast entry also 937 */ 938 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 939 nixlf, NIXLF_UCAST_ENTRY); 940 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 941 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 942 blkaddr, ucast_idx); 943 944 flow_key_alg = action.flow_key_alg; 945 if (action.op != NIX_RX_ACTIONOP_RSS) { 946 *(u64 *)&action = 0; 947 action.op = NIX_RX_ACTIONOP_UCAST; 948 action.pf_func = pcifunc; 949 } 950 951 /* RX_ACTION set to MCAST for CGX PF's */ 952 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) { 953 *(u64 *)&action = 0; 954 action.op = NIX_RX_ACTIONOP_MCAST; 955 action.index = pfvf->mcast_mce_idx; 956 } 957 958 mac_addr[0] = 0x01; /* LSB bit of 1st byte in DMAC */ 959 ether_addr_copy(req.packet.dmac, mac_addr); 960 ether_addr_copy(req.mask.dmac, mac_addr); 961 req.features = BIT_ULL(NPC_DMAC); 962 963 /* For cn10k the upper two bits of the channel number are 964 * cpt channel number. with masking out these bits in the 965 * mcam entry, same entry used for NIX will allow packets 966 * received from cpt for parsing. 967 */ 968 if (!is_rvu_otx2(rvu)) 969 req.chan_mask = NIX_CHAN_CPT_X2P_MASK; 970 else 971 req.chan_mask = 0xFFFU; 972 973 req.channel = chan; 974 req.intf = pfvf->nix_rx_intf; 975 req.entry = index; 976 req.op = action.op; 977 req.hdr.pcifunc = 0; /* AF is requester */ 978 req.vf = pcifunc | vf_func; 979 req.index = action.index; 980 req.match_id = action.match_id; 981 req.flow_key_alg = flow_key_alg; 982 983 if (is_cn20k(rvu->pdev)) 984 req.hw_prio = pfvf->hw_prio; 985 986 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 987 } 988 989 void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 990 bool enable) 991 { 992 struct npc_mcam *mcam = &rvu->hw->mcam; 993 int blkaddr, index; 994 995 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 996 if (blkaddr < 0) 997 return; 998 999 /* Get 'pcifunc' of PF device */ 1000 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 1001 1002 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 1003 NIXLF_ALLMULTI_ENTRY); 1004 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1005 } 1006 1007 static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, 1008 int blkaddr, u16 pcifunc, u64 rx_action) 1009 { 1010 int actindex, index, bank, entry; 1011 struct rvu_npc_mcam_rule *rule; 1012 bool enable, update; 1013 1014 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 1015 return; 1016 1017 mutex_lock(&mcam->lock); 1018 for (index = 0; index < mcam->bmap_entries; index++) { 1019 if (mcam->entry2target_pffunc[index] != pcifunc) 1020 continue; 1021 update = true; 1022 /* update not needed for the rules added via ntuple filters */ 1023 list_for_each_entry(rule, &mcam->mcam_rules, list) { 1024 if (rule->entry == index) 1025 update = false; 1026 } 1027 if (!update) 1028 continue; 1029 bank = npc_get_bank(mcam, index); 1030 actindex = index; 1031 entry = index & (mcam->banksize - 1); 1032 1033 /* read vf flow entry enable status */ 1034 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, 1035 actindex); 1036 /* disable before mcam entry update */ 1037 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, 1038 false); 1039 1040 /* update 'action' */ 1041 if (is_cn20k(rvu->pdev)) 1042 rvu_write64(rvu, blkaddr, 1043 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(entry, 1044 bank, 1045 0), 1046 rx_action); 1047 else 1048 rvu_write64(rvu, blkaddr, 1049 NPC_AF_MCAMEX_BANKX_ACTION(entry, bank), 1050 rx_action); 1051 1052 if (enable) 1053 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1054 actindex, true); 1055 } 1056 mutex_unlock(&mcam->lock); 1057 } 1058 1059 static void npc_update_rx_action_with_alg_idx(struct rvu *rvu, struct nix_rx_action action, 1060 struct rvu_pfvf *pfvf, int mcam_index, int blkaddr, 1061 int alg_idx) 1062 1063 { 1064 struct npc_mcam *mcam = &rvu->hw->mcam; 1065 struct rvu_hwinfo *hw = rvu->hw; 1066 int bank, op_rss; 1067 u64 reg; 1068 1069 if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, mcam_index)) 1070 return; 1071 1072 op_rss = (!hw->cap.nix_rx_multicast || !pfvf->use_mce_list); 1073 1074 bank = npc_get_bank(mcam, mcam_index); 1075 mcam_index &= (mcam->banksize - 1); 1076 1077 if (is_cn20k(rvu->pdev)) 1078 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_index, 1079 bank, 0); 1080 else 1081 reg = NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank); 1082 1083 /* If Rx action is MCAST update only RSS algorithm index */ 1084 if (!op_rss) { 1085 *(u64 *)&action = rvu_read64(rvu, blkaddr, reg); 1086 1087 action.flow_key_alg = alg_idx; 1088 } 1089 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action); 1090 } 1091 1092 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 1093 int group, int alg_idx, int mcam_index) 1094 { 1095 struct npc_mcam *mcam = &rvu->hw->mcam; 1096 struct nix_rx_action action; 1097 int blkaddr, index, bank; 1098 struct rvu_pfvf *pfvf; 1099 u64 reg; 1100 1101 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1102 if (blkaddr < 0) 1103 return; 1104 1105 /* Check if this is for reserved default entry */ 1106 if (mcam_index < 0) { 1107 if (group != DEFAULT_RSS_CONTEXT_GROUP) 1108 return; 1109 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1110 nixlf, NIXLF_UCAST_ENTRY); 1111 } else { 1112 /* TODO: validate this mcam index */ 1113 index = mcam_index; 1114 } 1115 1116 if (index >= mcam->total_entries) 1117 return; 1118 1119 bank = npc_get_bank(mcam, index); 1120 index &= (mcam->banksize - 1); 1121 1122 if (is_cn20k(rvu->pdev)) 1123 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); 1124 else 1125 reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); 1126 1127 *(u64 *)&action = rvu_read64(rvu, blkaddr, reg); 1128 /* Ignore if no action was set earlier */ 1129 if (!*(u64 *)&action) 1130 return; 1131 1132 action.op = NIX_RX_ACTIONOP_RSS; 1133 action.pf_func = pcifunc; 1134 action.index = group; 1135 action.flow_key_alg = alg_idx; 1136 1137 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action); 1138 /* update the VF flow rule action with the VF default entry action */ 1139 if (mcam_index < 0) 1140 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 1141 *(u64 *)&action); 1142 1143 /* update the action change in default rule */ 1144 pfvf = rvu_get_pfvf(rvu, pcifunc); 1145 if (pfvf->def_ucast_rule) 1146 pfvf->def_ucast_rule->rx_action = action; 1147 1148 if (mcam_index < 0) { 1149 /* update the VF flow rule action with the VF default 1150 * entry action 1151 */ 1152 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 1153 *(u64 *)&action); 1154 1155 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1156 nixlf, NIXLF_PROMISC_ENTRY); 1157 1158 /* If PF's promiscuous entry is enabled, 1159 * Set RSS action for that entry as well 1160 */ 1161 npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, 1162 blkaddr, alg_idx); 1163 1164 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1165 nixlf, NIXLF_ALLMULTI_ENTRY); 1166 /* If PF's allmulti entry is enabled, 1167 * Set RSS action for that entry as well 1168 */ 1169 npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, 1170 blkaddr, alg_idx); 1171 } 1172 } 1173 1174 void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, 1175 int nixlf, int type, bool enable) 1176 { 1177 struct npc_mcam *mcam = &rvu->hw->mcam; 1178 struct rvu_hwinfo *hw = rvu->hw; 1179 struct nix_mce_list *mce_list; 1180 int index, blkaddr, mce_idx; 1181 struct rvu_pfvf *pfvf; 1182 1183 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1184 if (blkaddr < 0) 1185 return; 1186 1187 index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK, 1188 nixlf, type); 1189 1190 /* disable MCAM entry when packet replication is not supported by hw */ 1191 if (!hw->cap.nix_rx_multicast && !is_vf(pcifunc)) { 1192 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1193 return; 1194 } 1195 1196 /* return incase mce list is not enabled */ 1197 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 1198 if (hw->cap.nix_rx_multicast && is_vf(pcifunc) && 1199 type != NIXLF_BCAST_ENTRY && !pfvf->use_mce_list) 1200 return; 1201 1202 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); 1203 1204 nix_update_mce_list(rvu, pcifunc, mce_list, 1205 mce_idx, index, enable); 1206 if (enable) 1207 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1208 } 1209 1210 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 1211 int nixlf, bool enable) 1212 { 1213 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 1214 struct npc_mcam *mcam = &rvu->hw->mcam; 1215 int index, blkaddr; 1216 1217 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1218 if (blkaddr < 0) 1219 return; 1220 1221 /* Ucast MCAM match entry of this PF/VF */ 1222 if (npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), 1223 pfvf->nix_rx_intf)) { 1224 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1225 nixlf, NIXLF_UCAST_ENTRY); 1226 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1227 } 1228 1229 /* Nothing to do for VFs, on platforms where pkt replication 1230 * is not supported 1231 */ 1232 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !rvu->hw->cap.nix_rx_multicast) 1233 return; 1234 1235 /* add/delete pf_func to broadcast MCE list */ 1236 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1237 NIXLF_BCAST_ENTRY, enable); 1238 } 1239 1240 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1241 { 1242 if (nixlf < 0) 1243 return; 1244 1245 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 1246 1247 /* Delete multicast and promisc MCAM entries */ 1248 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1249 NIXLF_ALLMULTI_ENTRY, false); 1250 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1251 NIXLF_PROMISC_ENTRY, false); 1252 } 1253 1254 bool rvu_npc_enable_mcam_by_entry_index(struct rvu *rvu, int entry, int intf, bool enable) 1255 { 1256 int blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1257 struct npc_mcam *mcam = &rvu->hw->mcam; 1258 struct rvu_npc_mcam_rule *rule, *tmp; 1259 1260 mutex_lock(&mcam->lock); 1261 1262 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1263 if (rule->intf != intf) 1264 continue; 1265 1266 if (rule->entry != entry) 1267 continue; 1268 1269 rule->enable = enable; 1270 mutex_unlock(&mcam->lock); 1271 1272 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1273 entry, enable); 1274 1275 return true; 1276 } 1277 1278 mutex_unlock(&mcam->lock); 1279 return false; 1280 } 1281 1282 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1283 { 1284 if (nixlf < 0) 1285 return; 1286 1287 /* Enables only broadcast match entry. Promisc/Allmulti are enabled 1288 * in set_rx_mode mbox handler. 1289 */ 1290 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 1291 } 1292 1293 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1294 { 1295 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 1296 struct npc_mcam *mcam = &rvu->hw->mcam; 1297 struct rvu_npc_mcam_rule *rule, *tmp; 1298 int blkaddr; 1299 1300 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1301 if (blkaddr < 0) 1302 return; 1303 1304 mutex_lock(&mcam->lock); 1305 1306 /* Disable MCAM entries directing traffic to this 'pcifunc' */ 1307 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1308 if (is_npc_intf_rx(rule->intf) && 1309 rule->rx_action.pf_func == pcifunc && 1310 rule->rx_action.op != NIX_RX_ACTIONOP_MCAST) { 1311 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1312 rule->entry, false); 1313 rule->enable = false; 1314 /* Indicate that default rule is disabled */ 1315 if (rule->default_rule) { 1316 pfvf->def_ucast_rule = NULL; 1317 list_del(&rule->list); 1318 kfree(rule); 1319 } 1320 } 1321 } 1322 1323 mutex_unlock(&mcam->lock); 1324 1325 npc_mcam_disable_flows(rvu, pcifunc); 1326 1327 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1328 } 1329 1330 void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1331 { 1332 struct npc_mcam *mcam = &rvu->hw->mcam; 1333 struct rvu_npc_mcam_rule *rule, *tmp; 1334 int blkaddr; 1335 1336 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1337 if (blkaddr < 0) 1338 return; 1339 1340 mutex_lock(&mcam->lock); 1341 1342 /* Free all MCAM entries owned by this 'pcifunc' */ 1343 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1344 1345 /* Free all MCAM counters owned by this 'pcifunc' */ 1346 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 1347 1348 /* Delete MCAM entries owned by this 'pcifunc' */ 1349 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1350 if (rule->owner == pcifunc && !rule->default_rule) { 1351 list_del(&rule->list); 1352 kfree(rule); 1353 } 1354 } 1355 1356 mutex_unlock(&mcam->lock); 1357 1358 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1359 } 1360 1361 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 1362 struct npc_mcam_kex *mkex, u8 intf) 1363 { 1364 int lid, lt, ld, fl; 1365 1366 if (is_npc_intf_tx(intf)) 1367 return; 1368 1369 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1370 mkex->keyx_cfg[NIX_INTF_RX]); 1371 1372 /* Program LDATA */ 1373 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1374 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1375 for (ld = 0; ld < NPC_MAX_LD; ld++) 1376 SET_KEX_LD(intf, lid, lt, ld, 1377 mkex->intf_lid_lt_ld[NIX_INTF_RX] 1378 [lid][lt][ld]); 1379 } 1380 } 1381 /* Program LFLAGS */ 1382 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1383 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1384 SET_KEX_LDFLAGS(intf, ld, fl, 1385 mkex->intf_ld_flags[NIX_INTF_RX] 1386 [ld][fl]); 1387 } 1388 } 1389 1390 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 1391 struct npc_mcam_kex *mkex, u8 intf) 1392 { 1393 int lid, lt, ld, fl; 1394 1395 if (is_npc_intf_rx(intf)) 1396 return; 1397 1398 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1399 mkex->keyx_cfg[NIX_INTF_TX]); 1400 1401 /* Program LDATA */ 1402 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1403 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1404 for (ld = 0; ld < NPC_MAX_LD; ld++) 1405 SET_KEX_LD(intf, lid, lt, ld, 1406 mkex->intf_lid_lt_ld[NIX_INTF_TX] 1407 [lid][lt][ld]); 1408 } 1409 } 1410 /* Program LFLAGS */ 1411 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1412 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1413 SET_KEX_LDFLAGS(intf, ld, fl, 1414 mkex->intf_ld_flags[NIX_INTF_TX] 1415 [ld][fl]); 1416 } 1417 } 1418 1419 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 1420 struct npc_mcam_kex *mkex) 1421 { 1422 struct rvu_hwinfo *hw = rvu->hw; 1423 u8 intf; 1424 int ld; 1425 1426 for (ld = 0; ld < NPC_MAX_LD; ld++) 1427 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld), 1428 mkex->kex_ld_flags[ld]); 1429 1430 for (intf = 0; intf < hw->npc_intfs; intf++) { 1431 npc_program_mkex_rx(rvu, blkaddr, mkex, intf); 1432 npc_program_mkex_tx(rvu, blkaddr, mkex, intf); 1433 } 1434 1435 /* Programme mkex hash profile */ 1436 npc_program_mkex_hash(rvu, blkaddr); 1437 } 1438 1439 int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, 1440 u64 *size) 1441 { 1442 u64 prfl_addr, prfl_sz; 1443 1444 if (!rvu->fwdata) 1445 return -EINVAL; 1446 1447 prfl_addr = rvu->fwdata->mcam_addr; 1448 prfl_sz = rvu->fwdata->mcam_sz; 1449 1450 if (!prfl_addr || !prfl_sz) 1451 return -EINVAL; 1452 1453 *prfl_img_addr = ioremap_wc(prfl_addr, prfl_sz); 1454 if (!(*prfl_img_addr)) 1455 return -ENOMEM; 1456 1457 *size = prfl_sz; 1458 1459 return 0; 1460 } 1461 1462 /* strtoull of "mkexprof" with base:36 */ 1463 #define MKEX_END_SIGN 0xdeadbeef 1464 1465 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, 1466 const char *mkex_profile) 1467 { 1468 struct device *dev = &rvu->pdev->dev; 1469 struct npc_mcam_kex *mcam_kex; 1470 void __iomem *mkex_prfl_addr = NULL; 1471 u64 prfl_sz; 1472 int ret; 1473 1474 /* If user not selected mkex profile */ 1475 if (rvu->kpu_fwdata_sz || 1476 !strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) 1477 goto program_mkex; 1478 1479 /* Setting up the mapping for mkex profile image */ 1480 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 1481 if (ret < 0) 1482 goto program_mkex; 1483 1484 mcam_kex = (struct npc_mcam_kex __force *)mkex_prfl_addr; 1485 1486 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { 1487 /* Compare with mkex mod_param name string */ 1488 if (mcam_kex->mkex_sign == MKEX_SIGN && 1489 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) { 1490 /* Due to an errata (35786) in A0/B0 pass silicon, 1491 * parse nibble enable configuration has to be 1492 * identical for both Rx and Tx interfaces. 1493 */ 1494 if (!is_rvu_96xx_B0(rvu) || 1495 mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) 1496 rvu->kpu.mcam_kex_prfl.mkex = mcam_kex; 1497 goto program_mkex; 1498 } 1499 1500 mcam_kex++; 1501 prfl_sz -= sizeof(struct npc_mcam_kex); 1502 } 1503 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 1504 1505 program_mkex: 1506 dev_info(rvu->dev, "Using %s mkex profile\n", 1507 rvu->kpu.mcam_kex_prfl.mkex->name); 1508 /* Program selected mkex profile */ 1509 npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mcam_kex_prfl.mkex); 1510 if (mkex_prfl_addr) 1511 iounmap(mkex_prfl_addr); 1512 } 1513 1514 void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 1515 const struct npc_kpu_profile_action *kpuaction, 1516 int kpu, int entry, bool pkind) 1517 { 1518 struct npc_kpu_action0 action0 = {0}; 1519 struct npc_kpu_action1 action1 = {0}; 1520 u64 reg; 1521 1522 action1.errlev = kpuaction->errlev; 1523 action1.errcode = kpuaction->errcode; 1524 action1.dp0_offset = kpuaction->dp0_offset; 1525 action1.dp1_offset = kpuaction->dp1_offset; 1526 action1.dp2_offset = kpuaction->dp2_offset; 1527 1528 if (pkind) 1529 reg = NPC_AF_PKINDX_ACTION1(entry); 1530 else 1531 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 1532 1533 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 1534 1535 action0.byp_count = kpuaction->bypass_count; 1536 action0.capture_ena = kpuaction->cap_ena; 1537 action0.parse_done = kpuaction->parse_done; 1538 action0.next_state = kpuaction->next_state; 1539 action0.capture_lid = kpuaction->lid; 1540 action0.capture_ltype = kpuaction->ltype; 1541 action0.capture_flags = kpuaction->flags; 1542 action0.ptr_advance = kpuaction->ptr_advance; 1543 action0.var_len_offset = kpuaction->offset; 1544 action0.var_len_mask = kpuaction->mask; 1545 action0.var_len_right = kpuaction->right; 1546 action0.var_len_shift = kpuaction->shift; 1547 1548 if (pkind) 1549 reg = NPC_AF_PKINDX_ACTION0(entry); 1550 else 1551 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 1552 1553 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 1554 } 1555 1556 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 1557 const struct npc_kpu_profile_cam *kpucam, 1558 int kpu, int entry) 1559 { 1560 struct npc_kpu_cam cam0 = {0}; 1561 struct npc_kpu_cam cam1 = {0}; 1562 1563 cam1.state = kpucam->state & kpucam->state_mask; 1564 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 1565 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 1566 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 1567 1568 cam0.state = ~kpucam->state & kpucam->state_mask; 1569 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 1570 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 1571 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 1572 1573 rvu_write64(rvu, blkaddr, 1574 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 1575 rvu_write64(rvu, blkaddr, 1576 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 1577 } 1578 1579 u64 npc_enable_mask(int count) 1580 { 1581 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 1582 } 1583 1584 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 1585 const struct npc_kpu_profile *profile) 1586 { 1587 int entry, num_entries, max_entries; 1588 u64 entry_mask; 1589 1590 if (profile->cam_entries != profile->action_entries) { 1591 dev_err(rvu->dev, 1592 "KPU%d: CAM and action entries [%d != %d] not equal\n", 1593 kpu, profile->cam_entries, profile->action_entries); 1594 } 1595 1596 max_entries = rvu->hw->npc_kpu_entries; 1597 1598 /* Program CAM match entries for previous KPU extracted data */ 1599 num_entries = min_t(int, profile->cam_entries, max_entries); 1600 for (entry = 0; entry < num_entries; entry++) 1601 npc_config_kpucam(rvu, blkaddr, 1602 &profile->cam[entry], kpu, entry); 1603 1604 /* Program this KPU's actions */ 1605 num_entries = min_t(int, profile->action_entries, max_entries); 1606 for (entry = 0; entry < num_entries; entry++) 1607 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry], 1608 kpu, entry, false); 1609 1610 /* Enable all programmed entries */ 1611 num_entries = min_t(int, profile->action_entries, profile->cam_entries); 1612 entry_mask = npc_enable_mask(num_entries); 1613 /* Disable first KPU_MAX_CST_ENT entries for built-in profile */ 1614 if (!rvu->kpu.custom) 1615 entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0); 1616 rvu_write64(rvu, blkaddr, 1617 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), entry_mask); 1618 if (num_entries > 64) { 1619 rvu_write64(rvu, blkaddr, 1620 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 1621 npc_enable_mask(num_entries - 64)); 1622 } 1623 1624 /* Enable this KPU */ 1625 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 1626 } 1627 1628 static void npc_prepare_default_kpu(struct rvu *rvu, 1629 struct npc_kpu_profile_adapter *profile) 1630 { 1631 profile->custom = 0; 1632 profile->name = def_pfl_name; 1633 profile->version = NPC_KPU_PROFILE_VER; 1634 profile->ikpu = ikpu_action_entries; 1635 profile->pkinds = ARRAY_SIZE(ikpu_action_entries); 1636 profile->kpu = npc_kpu_profiles; 1637 profile->kpus = ARRAY_SIZE(npc_kpu_profiles); 1638 profile->lt_def = &npc_lt_defaults; 1639 profile->mkex_hash = &npc_mkex_hash_default; 1640 1641 if (!is_cn20k(rvu->pdev)) { 1642 profile->mcam_kex_prfl.mkex = &npc_mkex_default; 1643 return; 1644 } 1645 1646 profile->mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get(); 1647 ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].offset = 6; 1648 ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].mask = 0xe0; 1649 ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].shift = 0x5; 1650 ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].right = 0x1; 1651 1652 npc_cn20k_update_action_entries_n_flags(rvu, profile); 1653 } 1654 1655 static int npc_apply_custom_kpu(struct rvu *rvu, 1656 struct npc_kpu_profile_adapter *profile) 1657 { 1658 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; 1659 struct npc_kpu_profile_action *action; 1660 struct npc_kpu_profile_fwdata *fw; 1661 struct npc_kpu_profile_cam *cam; 1662 struct npc_kpu_fwdata *fw_kpu; 1663 int entries; 1664 u16 kpu, entry; 1665 1666 if (is_cn20k(rvu->pdev)) 1667 return npc_cn20k_apply_custom_kpu(rvu, profile); 1668 1669 fw = rvu->kpu_fwdata; 1670 1671 if (rvu->kpu_fwdata_sz < hdr_sz) { 1672 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 1673 return -EINVAL; 1674 } 1675 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 1676 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 1677 fw->signature); 1678 return -EINVAL; 1679 } 1680 /* Verify if the using known profile structure */ 1681 if (NPC_KPU_VER_MAJ(profile->version) > 1682 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 1683 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 1684 NPC_KPU_VER_MAJ(profile->version), 1685 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 1686 return -EINVAL; 1687 } 1688 /* Verify if profile is aligned with the required kernel changes */ 1689 if (NPC_KPU_VER_MIN(profile->version) < 1690 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 1691 dev_warn(rvu->dev, 1692 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 1693 NPC_KPU_VER_MAJ(profile->version), 1694 NPC_KPU_VER_MIN(profile->version), 1695 NPC_KPU_VER_PATCH(profile->version), 1696 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 1697 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 1698 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 1699 return -EINVAL; 1700 } 1701 /* Verify if profile fits the HW */ 1702 if (fw->kpus > profile->kpus) { 1703 dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus, 1704 profile->kpus); 1705 return -EINVAL; 1706 } 1707 1708 profile->custom = 1; 1709 profile->name = fw->name; 1710 profile->version = le64_to_cpu(fw->version); 1711 profile->mcam_kex_prfl.mkex = &fw->mkex; 1712 profile->lt_def = &fw->lt_def; 1713 1714 for (kpu = 0; kpu < fw->kpus; kpu++) { 1715 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1716 if (fw_kpu->entries > KPU_MAX_CST_ENT) 1717 dev_warn(rvu->dev, 1718 "Too many custom entries on KPU%d: %d > %d\n", 1719 kpu, fw_kpu->entries, KPU_MAX_CST_ENT); 1720 entries = min(fw_kpu->entries, KPU_MAX_CST_ENT); 1721 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1722 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1723 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1724 offset += fw_kpu->entries * sizeof(*action); 1725 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1726 dev_warn(rvu->dev, 1727 "Profile size mismatch on KPU%i parsing.\n", 1728 kpu + 1); 1729 return -EINVAL; 1730 } 1731 for (entry = 0; entry < entries; entry++) { 1732 profile->kpu[kpu].cam[entry] = cam[entry]; 1733 profile->kpu[kpu].action[entry] = action[entry]; 1734 } 1735 } 1736 1737 return 0; 1738 } 1739 1740 static int npc_load_kpu_prfl_img(struct rvu *rvu, void __iomem *prfl_addr, 1741 u64 prfl_sz, const char *kpu_profile) 1742 { 1743 struct npc_kpu_profile_fwdata *kpu_data = NULL; 1744 int rc = -EINVAL; 1745 1746 kpu_data = (struct npc_kpu_profile_fwdata __force *)prfl_addr; 1747 if (le64_to_cpu(kpu_data->signature) == KPU_SIGN && 1748 !strncmp(kpu_data->name, kpu_profile, KPU_NAME_LEN)) { 1749 dev_info(rvu->dev, "Loading KPU profile from firmware db: %s\n", 1750 kpu_profile); 1751 rvu->kpu_fwdata = kpu_data; 1752 rvu->kpu_fwdata_sz = prfl_sz; 1753 rvu->kpu_prfl_addr = prfl_addr; 1754 rc = 0; 1755 } 1756 1757 return rc; 1758 } 1759 1760 static int npc_fwdb_detect_load_prfl_img(struct rvu *rvu, uint64_t prfl_sz, 1761 const char *kpu_profile) 1762 { 1763 struct npc_coalesced_kpu_prfl *img_data = NULL; 1764 int i = 0, rc = -EINVAL; 1765 void __iomem *kpu_prfl_addr; 1766 u32 offset; 1767 1768 img_data = (struct npc_coalesced_kpu_prfl __force *)rvu->kpu_prfl_addr; 1769 if (le64_to_cpu(img_data->signature) == KPU_SIGN && 1770 !strncmp(img_data->name, kpu_profile, KPU_NAME_LEN)) { 1771 /* Loaded profile is a single KPU profile. */ 1772 rc = npc_load_kpu_prfl_img(rvu, rvu->kpu_prfl_addr, 1773 prfl_sz, kpu_profile); 1774 goto done; 1775 } 1776 1777 /* Loaded profile is coalesced image, offset of first KPU profile.*/ 1778 offset = offsetof(struct npc_coalesced_kpu_prfl, prfl_sz) + 1779 (img_data->num_prfl * sizeof(uint16_t)); 1780 /* Check if mapped image is coalesced image. */ 1781 while (i < img_data->num_prfl) { 1782 /* Profile image offsets are rounded up to next 8 multiple.*/ 1783 offset = ALIGN_8B_CEIL(offset); 1784 kpu_prfl_addr = (void __iomem *)((uintptr_t)rvu->kpu_prfl_addr + 1785 offset); 1786 rc = npc_load_kpu_prfl_img(rvu, kpu_prfl_addr, 1787 img_data->prfl_sz[i], kpu_profile); 1788 if (!rc) 1789 break; 1790 /* Calculating offset of profile image based on profile size.*/ 1791 offset += img_data->prfl_sz[i]; 1792 i++; 1793 } 1794 done: 1795 return rc; 1796 } 1797 1798 static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile) 1799 { 1800 int ret = -EINVAL; 1801 u64 prfl_sz; 1802 1803 /* Setting up the mapping for NPC profile image */ 1804 ret = npc_fwdb_prfl_img_map(rvu, &rvu->kpu_prfl_addr, &prfl_sz); 1805 if (ret < 0) 1806 goto done; 1807 1808 /* Detect if profile is coalesced or single KPU profile and load */ 1809 ret = npc_fwdb_detect_load_prfl_img(rvu, prfl_sz, kpu_profile); 1810 if (ret == 0) 1811 goto done; 1812 1813 /* Cleaning up if KPU profile image from fwdata is not valid. */ 1814 if (rvu->kpu_prfl_addr) { 1815 iounmap(rvu->kpu_prfl_addr); 1816 rvu->kpu_prfl_addr = NULL; 1817 rvu->kpu_fwdata_sz = 0; 1818 rvu->kpu_fwdata = NULL; 1819 } 1820 1821 done: 1822 return ret; 1823 } 1824 1825 void npc_load_kpu_profile(struct rvu *rvu) 1826 { 1827 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1828 const char *kpu_profile = rvu->kpu_pfl_name; 1829 const struct firmware *fw = NULL; 1830 bool retry_fwdb = false; 1831 1832 /* If user not specified profile customization */ 1833 if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN)) 1834 goto revert_to_default; 1835 /* First prepare default KPU, then we'll customize top entries. */ 1836 npc_prepare_default_kpu(rvu, profile); 1837 1838 /* Order of preceedence for load loading NPC profile (high to low) 1839 * Firmware binary in filesystem. 1840 * Firmware database method. 1841 * Default KPU profile. 1842 */ 1843 if (!request_firmware_direct(&fw, kpu_profile, rvu->dev)) { 1844 dev_info(rvu->dev, "Loading KPU profile from firmware: %s\n", 1845 kpu_profile); 1846 rvu->kpu_fwdata = kzalloc(fw->size, GFP_KERNEL); 1847 if (rvu->kpu_fwdata) { 1848 memcpy(rvu->kpu_fwdata, fw->data, fw->size); 1849 rvu->kpu_fwdata_sz = fw->size; 1850 } 1851 release_firmware(fw); 1852 retry_fwdb = true; 1853 goto program_kpu; 1854 } 1855 1856 load_image_fwdb: 1857 /* Loading the KPU profile using firmware database */ 1858 if (npc_load_kpu_profile_fwdb(rvu, kpu_profile)) 1859 goto revert_to_default; 1860 1861 program_kpu: 1862 /* Apply profile customization if firmware was loaded. */ 1863 if (!rvu->kpu_fwdata_sz || npc_apply_custom_kpu(rvu, profile)) { 1864 /* If image from firmware filesystem fails to load or invalid 1865 * retry with firmware database method. 1866 */ 1867 if (rvu->kpu_fwdata || rvu->kpu_fwdata_sz) { 1868 /* Loading image from firmware database failed. */ 1869 if (rvu->kpu_prfl_addr) { 1870 iounmap(rvu->kpu_prfl_addr); 1871 rvu->kpu_prfl_addr = NULL; 1872 } else { 1873 kfree(rvu->kpu_fwdata); 1874 } 1875 rvu->kpu_fwdata = NULL; 1876 rvu->kpu_fwdata_sz = 0; 1877 if (retry_fwdb) { 1878 retry_fwdb = false; 1879 goto load_image_fwdb; 1880 } 1881 } 1882 1883 dev_warn(rvu->dev, 1884 "Can't load KPU profile %s. Using default.\n", 1885 kpu_profile); 1886 kfree(rvu->kpu_fwdata); 1887 rvu->kpu_fwdata = NULL; 1888 goto revert_to_default; 1889 } 1890 1891 dev_info(rvu->dev, "Using custom profile '%s', version %d.%d.%d\n", 1892 profile->name, NPC_KPU_VER_MAJ(profile->version), 1893 NPC_KPU_VER_MIN(profile->version), 1894 NPC_KPU_VER_PATCH(profile->version)); 1895 1896 return; 1897 1898 revert_to_default: 1899 npc_prepare_default_kpu(rvu, profile); 1900 } 1901 1902 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 1903 { 1904 struct rvu_hwinfo *hw = rvu->hw; 1905 int num_pkinds, num_kpus, idx; 1906 1907 /* Disable all KPUs and their entries */ 1908 for (idx = 0; idx < hw->npc_kpus; idx++) { 1909 rvu_write64(rvu, blkaddr, 1910 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 1911 rvu_write64(rvu, blkaddr, 1912 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 1913 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 1914 } 1915 1916 /* Load and customize KPU profile. */ 1917 npc_load_kpu_profile(rvu); 1918 1919 /* First program IKPU profile i.e PKIND configs. 1920 * Check HW max count to avoid configuring junk or 1921 * writing to unsupported CSR addresses. 1922 */ 1923 num_pkinds = rvu->kpu.pkinds; 1924 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); 1925 1926 for (idx = 0; idx < num_pkinds; idx++) 1927 npc_config_kpuaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 0, idx, true); 1928 1929 /* Program KPU CAM and Action profiles */ 1930 num_kpus = rvu->kpu.kpus; 1931 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 1932 1933 for (idx = 0; idx < num_kpus; idx++) 1934 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); 1935 } 1936 1937 void npc_mcam_rsrcs_deinit(struct rvu *rvu) 1938 { 1939 struct npc_mcam *mcam = &rvu->hw->mcam; 1940 1941 bitmap_free(mcam->bmap); 1942 bitmap_free(mcam->bmap_reverse); 1943 kfree(mcam->entry2pfvf_map); 1944 kfree(mcam->cntr2pfvf_map); 1945 kfree(mcam->entry2cntr_map); 1946 kfree(mcam->cntr_refcnt); 1947 kfree(mcam->entry2target_pffunc); 1948 kfree(mcam->counters.bmap); 1949 } 1950 1951 int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 1952 { 1953 int nixlf_count = rvu_get_nixlf_count(rvu); 1954 struct npc_mcam *mcam = &rvu->hw->mcam; 1955 int rsvd, err; 1956 u16 index; 1957 int cntr; 1958 u64 cfg; 1959 1960 cfg = (rvu_read64(rvu, blkaddr, 1961 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 1962 mcam->keysize = cfg; 1963 1964 /* Number of banks combined per MCAM entry */ 1965 if (is_cn20k(rvu->pdev)) { 1966 /* In cn20k, x2 entries is allowed for x4 profile. 1967 * set total_entries as 8192 * 2 and key size as x2. 1968 */ 1969 mcam->total_entries = mcam->banks * mcam->banksize; 1970 if (cfg == NPC_MCAM_KEY_X2) 1971 mcam->banks_per_entry = 1; 1972 else 1973 mcam->banks_per_entry = 2; 1974 1975 rsvd = 0; 1976 } else { 1977 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * 1978 mcam->banksize; 1979 1980 if (cfg == NPC_MCAM_KEY_X4) 1981 mcam->banks_per_entry = 4; 1982 else if (cfg == NPC_MCAM_KEY_X2) 1983 mcam->banks_per_entry = 2; 1984 else 1985 mcam->banks_per_entry = 1; 1986 1987 /* Reserve one MCAM entry for each of the NIX LF to 1988 * guarantee space to install default matching DMAC rule. 1989 * Also reserve 2 MCAM entries for each PF for default 1990 * channel based matching or 'bcast & promisc' matching to 1991 * support BCAST and PROMISC modes of operation for PFs. 1992 * PF0 is excluded. 1993 */ 1994 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 1995 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 1996 if (mcam->total_entries <= rsvd) { 1997 dev_warn(rvu->dev, 1998 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 1999 mcam->total_entries); 2000 return -ENOMEM; 2001 } 2002 } 2003 2004 mcam->bmap_entries = mcam->total_entries - rsvd; 2005 /* cn20k does not need offsets to alloc mcam entries */ 2006 if (!is_cn20k(rvu->pdev)) { 2007 mcam->nixlf_offset = mcam->bmap_entries; 2008 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 2009 } 2010 2011 /* Allocate bitmaps for managing MCAM entries */ 2012 mcam->bmap = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL); 2013 if (!mcam->bmap) 2014 return -ENOMEM; 2015 2016 mcam->bmap_reverse = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL); 2017 if (!mcam->bmap_reverse) 2018 goto free_bmap; 2019 2020 mcam->bmap_fcnt = mcam->bmap_entries; 2021 2022 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 2023 mcam->entry2pfvf_map = kcalloc(mcam->bmap_entries, sizeof(u16), 2024 GFP_KERNEL); 2025 2026 if (!mcam->entry2pfvf_map) 2027 goto free_bmap_reverse; 2028 2029 /* Reserve 1/8th of MCAM entries at the bottom for low priority 2030 * allocations and another 1/8th at the top for high priority 2031 * allocations. 2032 */ 2033 if (!is_cn20k(rvu->pdev)) { 2034 mcam->lprio_count = mcam->bmap_entries / 8; 2035 if (mcam->lprio_count > BITS_PER_LONG) 2036 mcam->lprio_count = round_down(mcam->lprio_count, 2037 BITS_PER_LONG); 2038 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 2039 mcam->hprio_count = mcam->lprio_count; 2040 mcam->hprio_end = mcam->hprio_count; 2041 } 2042 2043 /* Allocate bitmap for managing MCAM counters and memory 2044 * for saving counter to RVU PFFUNC allocation mapping. 2045 */ 2046 err = rvu_alloc_bitmap(&mcam->counters); 2047 if (err) 2048 goto free_entry_map; 2049 2050 mcam->cntr2pfvf_map = kcalloc(mcam->counters.max, sizeof(u16), 2051 GFP_KERNEL); 2052 if (!mcam->cntr2pfvf_map) 2053 goto free_cntr_bmap; 2054 2055 /* Alloc memory for MCAM entry to counter mapping and for tracking 2056 * counter's reference count. 2057 */ 2058 mcam->entry2cntr_map = kcalloc(mcam->bmap_entries, sizeof(u16), 2059 GFP_KERNEL); 2060 if (!mcam->entry2cntr_map) 2061 goto free_cntr_map; 2062 2063 mcam->cntr_refcnt = kcalloc(mcam->counters.max, sizeof(u16), 2064 GFP_KERNEL); 2065 if (!mcam->cntr_refcnt) 2066 goto free_entry_cntr_map; 2067 2068 /* Alloc memory for saving target device of mcam rule */ 2069 mcam->entry2target_pffunc = kmalloc_array(mcam->total_entries, 2070 sizeof(u16), GFP_KERNEL); 2071 if (!mcam->entry2target_pffunc) 2072 goto free_cntr_refcnt; 2073 2074 for (index = 0; index < mcam->bmap_entries; index++) { 2075 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 2076 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 2077 } 2078 2079 for (cntr = 0; cntr < mcam->counters.max; cntr++) 2080 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 2081 2082 mutex_init(&mcam->lock); 2083 2084 return 0; 2085 2086 free_cntr_refcnt: 2087 kfree(mcam->cntr_refcnt); 2088 free_entry_cntr_map: 2089 kfree(mcam->entry2cntr_map); 2090 free_cntr_map: 2091 kfree(mcam->cntr2pfvf_map); 2092 free_cntr_bmap: 2093 kfree(mcam->counters.bmap); 2094 free_entry_map: 2095 kfree(mcam->entry2pfvf_map); 2096 free_bmap_reverse: 2097 bitmap_free(mcam->bmap_reverse); 2098 free_bmap: 2099 bitmap_free(mcam->bmap); 2100 2101 return -ENOMEM; 2102 } 2103 2104 static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) 2105 { 2106 struct npc_pkind *pkind = &rvu->hw->pkind; 2107 struct npc_mcam *mcam = &rvu->hw->mcam; 2108 struct rvu_hwinfo *hw = rvu->hw; 2109 u64 npc_const, npc_const1; 2110 u64 npc_const2 = 0; 2111 2112 npc_const = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 2113 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 2114 if (npc_const1 & BIT_ULL(63)) 2115 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 2116 2117 pkind->rsrc.max = NPC_UNRESERVED_PKIND_COUNT; 2118 hw->npc_pkinds = (npc_const1 >> 12) & 0xFFULL; 2119 hw->npc_kpu_entries = npc_const1 & 0xFFFULL; 2120 hw->npc_kpus = (npc_const >> 8) & 0x1FULL; 2121 /* For Cn20k silicon, check if enhanced parser 2122 * is present, then set the NUM_KPMS = NUM_KPUS / 2 and 2123 * number of LDATA extractors per KEX. 2124 */ 2125 if (is_cn20k(rvu->pdev)) { 2126 if (!(npc_const1 & BIT_ULL(62))) { 2127 WARN(1, "Enhanced parser is not supported\n"); 2128 return; 2129 } 2130 hw->npc_kpms = hw->npc_kpus / 2; 2131 hw->npc_kex_extr = (npc_const1 >> 36) & 0x3FULL; 2132 } 2133 2134 hw->npc_intfs = npc_const & 0xFULL; 2135 hw->npc_counters = (npc_const >> 48) & 0xFFFFULL; 2136 2137 mcam->banks = (npc_const >> 44) & 0xFULL; 2138 mcam->banksize = (npc_const >> 28) & 0xFFFFULL; 2139 hw->npc_stat_ena = BIT_ULL(9); 2140 /* Extended set */ 2141 if (npc_const2) { 2142 hw->npc_ext_set = true; 2143 /* 96xx supports only match_stats and npc_counters 2144 * reflected in NPC_AF_CONST reg. 2145 * STAT_SEL and ENA are at [0:8] and 9 bit positions. 2146 * 98xx has both match_stat and ext and npc_counter 2147 * reflected in NPC_AF_CONST2 2148 * STAT_SEL_EXT added at [12:14] bit position. 2149 * cn10k supports only ext and hence npc_counters in 2150 * NPC_AF_CONST is 0 and npc_counters reflected in NPC_AF_CONST2. 2151 * STAT_SEL bitpos incremented from [0:8] to [0:11] and ENA bit moved to 63 2152 */ 2153 if (!hw->npc_counters) 2154 hw->npc_stat_ena = BIT_ULL(63); 2155 hw->npc_counters = (npc_const2 >> 16) & 0xFFFFULL; 2156 mcam->banksize = npc_const2 & 0xFFFFULL; 2157 } 2158 2159 mcam->counters.max = hw->npc_counters; 2160 } 2161 2162 static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) 2163 { 2164 struct npc_mcam_kex_extr *mkex_extr = rvu->kpu.mcam_kex_prfl.mkex_extr; 2165 struct npc_mcam_kex *mkex = rvu->kpu.mcam_kex_prfl.mkex; 2166 struct npc_mcam *mcam = &rvu->hw->mcam; 2167 struct rvu_hwinfo *hw = rvu->hw; 2168 u64 nibble_ena, rx_kex, tx_kex; 2169 u64 *keyx_cfg, reg; 2170 u8 intf; 2171 2172 if (is_cn20k(rvu->pdev)) { 2173 keyx_cfg = mkex_extr->keyx_cfg; 2174 } else { 2175 keyx_cfg = mkex->keyx_cfg; 2176 /* Reserve last counter for MCAM RX miss action which is set to 2177 * drop packet. This way we will know how many pkts didn't 2178 * match any MCAM entry. 2179 */ 2180 mcam->counters.max--; 2181 mcam->rx_miss_act_cntr = mcam->counters.max; 2182 } 2183 2184 rx_kex = keyx_cfg[NIX_INTF_RX]; 2185 tx_kex = keyx_cfg[NIX_INTF_TX]; 2186 2187 nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); 2188 2189 nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); 2190 if (nibble_ena) { 2191 tx_kex &= ~NPC_PARSE_NIBBLE; 2192 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); 2193 keyx_cfg[NIX_INTF_TX] = tx_kex; 2194 } 2195 2196 /* Configure RX interfaces */ 2197 for (intf = 0; intf < hw->npc_intfs; intf++) { 2198 if (is_npc_intf_tx(intf)) 2199 continue; 2200 2201 /* Set RX MCAM search key size. LA..LE (ltype only) + Channel */ 2202 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 2203 rx_kex); 2204 2205 if (is_cn20k(rvu->pdev)) 2206 reg = NPC_AF_INTFX_MISS_ACTX(intf, 0); 2207 else 2208 reg = NPC_AF_INTFX_MISS_ACT(intf); 2209 2210 /* If MCAM lookup doesn't result in a match, drop the received 2211 * packet. And map this action to a counter to count dropped 2212 * packets. 2213 */ 2214 rvu_write64(rvu, blkaddr, reg, NIX_RX_ACTIONOP_DROP); 2215 2216 if (is_cn20k(rvu->pdev)) 2217 continue; 2218 2219 /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9] 2220 * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0] 2221 */ 2222 rvu_write64(rvu, blkaddr, 2223 NPC_AF_INTFX_MISS_STAT_ACT(intf), 2224 ((mcam->rx_miss_act_cntr >> 9) << 12) | 2225 hw->npc_stat_ena | mcam->rx_miss_act_cntr); 2226 } 2227 2228 /* Configure TX interfaces */ 2229 for (intf = 0; intf < hw->npc_intfs; intf++) { 2230 if (is_npc_intf_rx(intf)) 2231 continue; 2232 2233 /* Extract Ltypes LID_LA to LID_LE */ 2234 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 2235 tx_kex); 2236 2237 if (is_cn20k(rvu->pdev)) 2238 reg = NPC_AF_INTFX_MISS_ACTX(intf, 0); 2239 else 2240 reg = NPC_AF_INTFX_MISS_ACT(intf); 2241 2242 /* Set TX miss action to UCAST_DEFAULT i.e 2243 * transmit the packet on NIX LF SQ's default channel. 2244 */ 2245 rvu_write64(rvu, blkaddr, reg, NIX_TX_ACTIONOP_UCAST_DEFAULT); 2246 } 2247 } 2248 2249 int rvu_npc_init(struct rvu *rvu) 2250 { 2251 struct npc_kpu_profile_adapter *kpu = &rvu->kpu; 2252 struct npc_pkind *pkind = &rvu->hw->pkind; 2253 struct npc_mcam *mcam = &rvu->hw->mcam; 2254 int blkaddr, entry, bank, err; 2255 2256 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2257 if (blkaddr < 0) { 2258 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 2259 return -ENODEV; 2260 } 2261 2262 rvu_npc_hw_init(rvu, blkaddr); 2263 2264 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 2265 for (bank = 0; bank < mcam->banks; bank++) { 2266 for (entry = 0; entry < mcam->banksize; entry++) 2267 rvu_write64(rvu, blkaddr, 2268 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 2269 } 2270 2271 err = rvu_alloc_bitmap(&pkind->rsrc); 2272 if (err) 2273 return err; 2274 /* Reserve PKIND#0 for LBKs. Power reset value of LBK_CH_PKIND is '0', 2275 * no need to configure PKIND for all LBKs separately. 2276 */ 2277 rvu_alloc_rsrc(&pkind->rsrc); 2278 2279 /* Allocate mem for pkind to PF and channel mapping info */ 2280 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 2281 sizeof(u32), GFP_KERNEL); 2282 if (!pkind->pfchan_map) 2283 return -ENOMEM; 2284 2285 /* Configure KPU profile */ 2286 if (is_cn20k(rvu->pdev)) 2287 npc_cn20k_parser_profile_init(rvu, blkaddr); 2288 else 2289 npc_parser_profile_init(rvu, blkaddr); 2290 2291 /* Config Outer L2, IPv4's NPC layer info */ 2292 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 2293 (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) | 2294 kpu->lt_def->pck_ol2.ltype_mask); 2295 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 2296 (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) | 2297 kpu->lt_def->pck_oip4.ltype_mask); 2298 2299 /* Config Inner IPV4 NPC layer info */ 2300 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, 2301 (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) | 2302 kpu->lt_def->pck_iip4.ltype_mask); 2303 2304 /* Enable below for Rx pkts. 2305 * - Outer IPv4 header checksum validation. 2306 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2B]. 2307 * - Detect outer L2 multicast address and set NPC_RESULT_S[L2M]. 2308 * - Inner IPv4 header checksum validation. 2309 * - Set non zero checksum error code value 2310 */ 2311 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 2312 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 2313 ((u64)NPC_EC_OIP4_CSUM << 32) | (NPC_EC_IIP4_CSUM << 24) | 2314 BIT_ULL(7) | BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1)); 2315 2316 rvu_npc_setup_interfaces(rvu, blkaddr); 2317 2318 npc_config_secret_key(rvu, blkaddr); 2319 /* Configure MKEX profile */ 2320 if (is_cn20k(rvu->pdev)) 2321 npc_cn20k_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 2322 else 2323 npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 2324 2325 err = npc_mcam_rsrcs_init(rvu, blkaddr); 2326 if (err) 2327 return err; 2328 2329 err = npc_flow_steering_init(rvu, blkaddr); 2330 if (err) { 2331 dev_err(rvu->dev, 2332 "Incorrect mkex profile loaded using default mkex\n"); 2333 if (is_cn20k(rvu->pdev)) 2334 npc_cn20k_load_mkex_profile(rvu, blkaddr, def_pfl_name); 2335 else 2336 npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); 2337 } 2338 2339 if (is_cn20k(rvu->pdev)) 2340 return npc_cn20k_init(rvu); 2341 2342 return 0; 2343 } 2344 2345 void rvu_npc_freemem(struct rvu *rvu) 2346 { 2347 struct npc_pkind *pkind = &rvu->hw->pkind; 2348 struct npc_mcam *mcam = &rvu->hw->mcam; 2349 2350 kfree(pkind->rsrc.bmap); 2351 npc_mcam_rsrcs_deinit(rvu); 2352 if (rvu->kpu_prfl_addr) 2353 iounmap(rvu->kpu_prfl_addr); 2354 else 2355 kfree(rvu->kpu_fwdata); 2356 mutex_destroy(&mcam->lock); 2357 2358 if (is_cn20k(rvu->pdev)) 2359 npc_cn20k_deinit(rvu); 2360 } 2361 2362 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, 2363 int blkaddr, int *alloc_cnt, 2364 int *enable_cnt) 2365 { 2366 struct npc_mcam *mcam = &rvu->hw->mcam; 2367 int entry; 2368 2369 *alloc_cnt = 0; 2370 *enable_cnt = 0; 2371 2372 for (entry = 0; entry < mcam->bmap_entries; entry++) { 2373 if (mcam->entry2pfvf_map[entry] == pcifunc) { 2374 (*alloc_cnt)++; 2375 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry)) 2376 (*enable_cnt)++; 2377 } 2378 } 2379 } 2380 2381 void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, 2382 int blkaddr, int *alloc_cnt, 2383 int *enable_cnt) 2384 { 2385 struct npc_mcam *mcam = &rvu->hw->mcam; 2386 int cntr; 2387 2388 *alloc_cnt = 0; 2389 *enable_cnt = 0; 2390 2391 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2392 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2393 (*alloc_cnt)++; 2394 if (mcam->cntr_refcnt[cntr]) 2395 (*enable_cnt)++; 2396 } 2397 } 2398 } 2399 2400 int npc_mcam_verify_entry(struct npc_mcam *mcam, 2401 u16 pcifunc, int entry) 2402 { 2403 /* verify AF installed entries */ 2404 if (is_pffunc_af(pcifunc)) 2405 return 0; 2406 /* Verify if entry is valid and if it is indeed 2407 * allocated to the requesting PFFUNC. 2408 */ 2409 if (entry >= mcam->bmap_entries) 2410 return NPC_MCAM_INVALID_REQ; 2411 2412 if (pcifunc != mcam->entry2pfvf_map[entry]) 2413 return NPC_MCAM_PERM_DENIED; 2414 2415 return 0; 2416 } 2417 2418 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 2419 u16 pcifunc, int cntr) 2420 { 2421 /* Verify if counter is valid and if it is indeed 2422 * allocated to the requesting PFFUNC. 2423 */ 2424 if (cntr >= mcam->counters.max) 2425 return NPC_MCAM_INVALID_REQ; 2426 2427 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 2428 return NPC_MCAM_PERM_DENIED; 2429 2430 return 0; 2431 } 2432 2433 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 2434 int blkaddr, u16 entry, u16 cntr) 2435 { 2436 u16 index = entry & (mcam->banksize - 1); 2437 u32 bank = npc_get_bank(mcam, entry); 2438 struct rvu_hwinfo *hw = rvu->hw; 2439 2440 /* Set mapping and increment counter's refcnt */ 2441 mcam->entry2cntr_map[entry] = cntr; 2442 mcam->cntr_refcnt[cntr]++; 2443 2444 if (is_cn20k(rvu->pdev)) 2445 return; 2446 2447 /* Enable stats */ 2448 rvu_write64(rvu, blkaddr, 2449 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 2450 ((cntr >> 9) << 12) | hw->npc_stat_ena | cntr); 2451 } 2452 2453 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 2454 struct npc_mcam *mcam, 2455 int blkaddr, u16 entry, u16 cntr) 2456 { 2457 u16 index = entry & (mcam->banksize - 1); 2458 u32 bank = npc_get_bank(mcam, entry); 2459 2460 /* Remove mapping and reduce counter's refcnt */ 2461 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 2462 mcam->cntr_refcnt[cntr]--; 2463 2464 if (is_cn20k(rvu->pdev)) 2465 return; 2466 2467 /* Disable stats */ 2468 rvu_write64(rvu, blkaddr, 2469 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 2470 } 2471 2472 /* Sets MCAM entry in bitmap as used. Update 2473 * reverse bitmap too. Should be called with 2474 * 'mcam->lock' held. 2475 */ 2476 void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 2477 { 2478 u16 entry, rentry; 2479 2480 entry = index; 2481 rentry = mcam->bmap_entries - index - 1; 2482 2483 __set_bit(entry, mcam->bmap); 2484 __set_bit(rentry, mcam->bmap_reverse); 2485 mcam->bmap_fcnt--; 2486 } 2487 2488 /* Sets MCAM entry in bitmap as free. Update 2489 * reverse bitmap too. Should be called with 2490 * 'mcam->lock' held. 2491 */ 2492 void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 2493 { 2494 u16 entry, rentry; 2495 2496 entry = index; 2497 rentry = mcam->bmap_entries - index - 1; 2498 2499 __clear_bit(entry, mcam->bmap); 2500 __clear_bit(rentry, mcam->bmap_reverse); 2501 mcam->bmap_fcnt++; 2502 } 2503 2504 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 2505 int blkaddr, u16 pcifunc) 2506 { 2507 u16 index, cntr; 2508 int rc; 2509 2510 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 2511 for (index = 0; index < mcam->bmap_entries; index++) { 2512 if (mcam->entry2pfvf_map[index] == pcifunc) { 2513 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 2514 /* Free the entry in bitmap */ 2515 npc_mcam_clear_bit(mcam, index); 2516 /* Disable the entry */ 2517 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 2518 2519 /* Update entry2counter mapping */ 2520 cntr = mcam->entry2cntr_map[index]; 2521 if (cntr != NPC_MCAM_INVALID_MAP) 2522 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 2523 blkaddr, index, 2524 cntr); 2525 mcam->entry2target_pffunc[index] = 0x0; 2526 if (is_cn20k(rvu->pdev)) { 2527 rc = npc_cn20k_idx_free(rvu, &index, 1); 2528 if (rc) 2529 dev_err(rvu->dev, 2530 "Failed to free mcam idx=%u pcifunc=%#x\n", 2531 index, pcifunc); 2532 } 2533 } 2534 } 2535 } 2536 2537 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 2538 u16 pcifunc) 2539 { 2540 u16 cntr; 2541 2542 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 2543 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2544 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2545 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 2546 mcam->cntr_refcnt[cntr] = 0; 2547 rvu_free_rsrc(&mcam->counters, cntr); 2548 /* This API is expected to be called after freeing 2549 * MCAM entries, which inturn will remove 2550 * 'entry to counter' mapping. 2551 * No need to do it again. 2552 */ 2553 } 2554 } 2555 } 2556 2557 /* Find area of contiguous free entries of size 'nr'. 2558 * If not found return max contiguous free entries available. 2559 */ 2560 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 2561 u16 nr, u16 *max_area) 2562 { 2563 u16 max_area_start = 0; 2564 u16 index, next, end; 2565 2566 *max_area = 0; 2567 2568 again: 2569 index = find_next_zero_bit(map, size, start); 2570 if (index >= size) 2571 return max_area_start; 2572 2573 end = ((index + nr) >= size) ? size : index + nr; 2574 next = find_next_bit(map, end, index); 2575 if (*max_area < (next - index)) { 2576 *max_area = next - index; 2577 max_area_start = index; 2578 } 2579 2580 if (next < end) { 2581 start = next + 1; 2582 goto again; 2583 } 2584 2585 return max_area_start; 2586 } 2587 2588 /* Find number of free MCAM entries available 2589 * within range i.e in between 'start' and 'end'. 2590 */ 2591 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 2592 { 2593 u16 index, next; 2594 u16 fcnt = 0; 2595 2596 again: 2597 if (start >= end) 2598 return fcnt; 2599 2600 index = find_next_zero_bit(map, end, start); 2601 if (index >= end) 2602 return fcnt; 2603 2604 next = find_next_bit(map, end, index); 2605 if (next <= end) { 2606 fcnt += next - index; 2607 start = next + 1; 2608 goto again; 2609 } 2610 2611 fcnt += end - index; 2612 return fcnt; 2613 } 2614 2615 static void 2616 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 2617 struct npc_mcam_alloc_entry_req *req, 2618 u16 *start, u16 *end, bool *reverse) 2619 { 2620 u16 fcnt; 2621 2622 if (req->ref_prio == NPC_MCAM_HIGHER_PRIO) 2623 goto hprio; 2624 2625 /* For a low priority entry allocation 2626 * - If reference entry is not in hprio zone then 2627 * search range: ref_entry to end. 2628 * - If reference entry is in hprio zone and if 2629 * request can be accomodated in non-hprio zone then 2630 * search range: 'start of middle zone' to 'end' 2631 * - else search in reverse, so that less number of hprio 2632 * zone entries are allocated. 2633 */ 2634 2635 *reverse = false; 2636 *start = req->ref_entry + 1; 2637 *end = mcam->bmap_entries; 2638 2639 if (req->ref_entry >= mcam->hprio_end) 2640 return; 2641 2642 fcnt = npc_mcam_get_free_count(mcam->bmap, 2643 mcam->hprio_end, mcam->bmap_entries); 2644 if (fcnt > req->count) 2645 *start = mcam->hprio_end; 2646 else 2647 *reverse = true; 2648 return; 2649 2650 hprio: 2651 /* For a high priority entry allocation, search is always 2652 * in reverse to preserve hprio zone entries. 2653 * - If reference entry is not in lprio zone then 2654 * search range: 0 to ref_entry. 2655 * - If reference entry is in lprio zone and if 2656 * request can be accomodated in middle zone then 2657 * search range: 'hprio_end' to 'lprio_start' 2658 */ 2659 2660 *reverse = true; 2661 *start = 0; 2662 *end = req->ref_entry; 2663 2664 if (req->ref_entry <= mcam->lprio_start) 2665 return; 2666 2667 fcnt = npc_mcam_get_free_count(mcam->bmap, 2668 mcam->hprio_end, mcam->lprio_start); 2669 if (fcnt < req->count) 2670 return; 2671 *start = mcam->hprio_end; 2672 *end = mcam->lprio_start; 2673 } 2674 2675 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 2676 struct npc_mcam_alloc_entry_req *req, 2677 struct npc_mcam_alloc_entry_rsp *rsp) 2678 { 2679 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 2680 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 2681 u16 fcnt, hp_fcnt, lp_fcnt; 2682 struct rvu *rvu = hw->rvu; 2683 u16 start, end, index; 2684 int entry, next_start; 2685 bool reverse = false; 2686 unsigned long *bmap; 2687 int ret, limit = 0; 2688 u16 max_contig; 2689 2690 if (!is_cn20k(rvu->pdev)) 2691 goto not_cn20k; 2692 2693 /* Only x2 or x4 key types are accepted */ 2694 if (req->kw_type != NPC_MCAM_KEY_X2 && req->kw_type != NPC_MCAM_KEY_X4) 2695 return NPC_MCAM_INVALID_REQ; 2696 2697 /* The below table is being followed during allocation, 2698 * 2699 * 1. ref_entry == 0 && prio == HIGH && count == 1 : user wants to 2700 * allocate 0th index 2701 * 2. ref_entry == 0 && prio == HIGH && count > 1 : Invalid request 2702 * 3. ref_entry == 0 && prio == LOW && count >= 1 : limit = 0 2703 * 4. ref_entry != 0 && prio == HIGH && count >= 1 : limit = 0 2704 * 5. ref_entry != 0 && prio == LOW && count >=1 : limit = Max 2705 * (X2 2*8192, X4 8192) 2706 */ 2707 if (req->ref_entry && req->ref_prio == NPC_MCAM_LOWER_PRIO) { 2708 if (req->kw_type == NPC_MCAM_KEY_X2) 2709 limit = 2 * mcam->bmap_entries; 2710 else 2711 limit = mcam->bmap_entries; 2712 } 2713 2714 ret = npc_cn20k_ref_idx_alloc(rvu, pcifunc, req->kw_type, 2715 req->ref_prio, rsp->entry_list, 2716 req->ref_entry, limit, 2717 req->contig, req->count, !!req->virt); 2718 2719 if (ret) { 2720 rsp->count = 0; 2721 return NPC_MCAM_ALLOC_FAILED; 2722 } 2723 2724 rsp->count = req->count; 2725 if (req->contig) 2726 rsp->entry = rsp->entry_list[0]; 2727 2728 /* cn20k, entries allocation algorithm is different. 2729 * This common API updates some bitmap on usage etc, which 2730 * will be used by other functions. So update those for 2731 * cn20k as well. 2732 */ 2733 2734 mutex_lock(&mcam->lock); 2735 /* Mark the allocated entries as used and set nixlf mapping */ 2736 for (entry = 0; entry < rsp->count; entry++) { 2737 index = npc_cn20k_vidx2idx(rsp->entry_list[entry]); 2738 npc_mcam_set_bit(mcam, index); 2739 mcam->entry2pfvf_map[index] = pcifunc; 2740 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 2741 } 2742 2743 /* cn20k, free count is provided thru different mbox message. 2744 * one counter to indicate free x2 slots and free x4 slots 2745 * does not provide any useful information to the user. 2746 */ 2747 rsp->free_count = -1; 2748 mutex_unlock(&mcam->lock); 2749 2750 return 0; 2751 2752 not_cn20k: 2753 mutex_lock(&mcam->lock); 2754 2755 /* Check if there are any free entries */ 2756 if (!mcam->bmap_fcnt) { 2757 mutex_unlock(&mcam->lock); 2758 return NPC_MCAM_ALLOC_FAILED; 2759 } 2760 2761 /* MCAM entries are divided into high priority, middle and 2762 * low priority zones. Idea is to not allocate top and lower 2763 * most entries as much as possible, this is to increase 2764 * probability of honouring priority allocation requests. 2765 * 2766 * Two bitmaps are used for mcam entry management, 2767 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 2768 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 2769 * 2770 * Reverse bitmap is used to allocate entries 2771 * - when a higher priority entry is requested 2772 * - when available free entries are less. 2773 * Lower priority ones out of avaialble free entries are always 2774 * chosen when 'high vs low' question arises. 2775 * 2776 * For a VF base MCAM match rule is set by its PF. And all the 2777 * further MCAM rules installed by VF on its own are 2778 * concatenated with the base rule set by its PF. Hence PF entries 2779 * should be at lower priority compared to VF entries. Otherwise 2780 * base rule is hit always and rules installed by VF will be of 2781 * no use. Hence if the request is from PF then allocate low 2782 * priority entries. 2783 */ 2784 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 2785 goto lprio_alloc; 2786 2787 /* Get the search range for priority allocation request */ 2788 if (req->ref_prio) { 2789 npc_get_mcam_search_range_priority(mcam, req, 2790 &start, &end, &reverse); 2791 goto alloc; 2792 } 2793 2794 /* Find out the search range for non-priority allocation request 2795 * 2796 * Get MCAM free entry count in middle zone. 2797 */ 2798 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 2799 mcam->lprio_start, 2800 mcam->bmap_entries); 2801 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 2802 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 2803 2804 /* Check if request can be accomodated in the middle zone */ 2805 if (fcnt > req->count) { 2806 start = mcam->hprio_end; 2807 end = mcam->lprio_start; 2808 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 2809 /* Expand search zone from half of hprio zone to 2810 * half of lprio zone. 2811 */ 2812 start = mcam->hprio_end / 2; 2813 end = mcam->bmap_entries - (mcam->lprio_count / 2); 2814 reverse = true; 2815 } else { 2816 /* Not enough free entries, search all entries in reverse, 2817 * so that low priority ones will get used up. 2818 */ 2819 lprio_alloc: 2820 reverse = true; 2821 start = 0; 2822 end = mcam->bmap_entries; 2823 /* Ensure PF requests are always at bottom and if PF requests 2824 * for higher/lower priority entry wrt reference entry then 2825 * honour that criteria and start search for entries from bottom 2826 * and not in mid zone. 2827 */ 2828 if (!(pcifunc & RVU_PFVF_FUNC_MASK) && 2829 req->ref_prio == NPC_MCAM_HIGHER_PRIO) 2830 end = req->ref_entry; 2831 2832 if (!(pcifunc & RVU_PFVF_FUNC_MASK) && 2833 req->ref_prio == NPC_MCAM_LOWER_PRIO) 2834 start = req->ref_entry; 2835 } 2836 2837 alloc: 2838 if (reverse) { 2839 bmap = mcam->bmap_reverse; 2840 start = mcam->bmap_entries - start; 2841 end = mcam->bmap_entries - end; 2842 swap(start, end); 2843 } else { 2844 bmap = mcam->bmap; 2845 } 2846 2847 if (req->contig) { 2848 /* Allocate requested number of contiguous entries, if 2849 * unsuccessful find max contiguous entries available. 2850 */ 2851 index = npc_mcam_find_zero_area(bmap, end, start, 2852 req->count, &max_contig); 2853 rsp->count = max_contig; 2854 if (reverse) 2855 rsp->entry = mcam->bmap_entries - index - max_contig; 2856 else 2857 rsp->entry = index; 2858 } else { 2859 /* Allocate requested number of non-contiguous entries, 2860 * if unsuccessful allocate as many as possible. 2861 */ 2862 rsp->count = 0; 2863 next_start = start; 2864 for (entry = 0; entry < req->count; entry++) { 2865 index = find_next_zero_bit(bmap, end, next_start); 2866 if (index >= end) 2867 break; 2868 2869 next_start = start + (index - start) + 1; 2870 2871 /* Save the entry's index */ 2872 if (reverse) 2873 index = mcam->bmap_entries - index - 1; 2874 entry_list[entry] = index; 2875 rsp->count++; 2876 } 2877 } 2878 2879 /* If allocating requested no of entries is unsucessful, 2880 * expand the search range to full bitmap length and retry. 2881 */ 2882 if (!req->ref_prio && rsp->count < req->count && 2883 ((end - start) != mcam->bmap_entries)) { 2884 reverse = true; 2885 start = 0; 2886 end = mcam->bmap_entries; 2887 goto alloc; 2888 } 2889 2890 /* For priority entry allocation requests, if allocation is 2891 * failed then expand search to max possible range and retry. 2892 */ 2893 if (req->ref_prio && rsp->count < req->count) { 2894 if (req->ref_prio == NPC_MCAM_LOWER_PRIO && 2895 (start != (req->ref_entry + 1))) { 2896 start = req->ref_entry + 1; 2897 end = mcam->bmap_entries; 2898 reverse = false; 2899 goto alloc; 2900 } else if ((req->ref_prio == NPC_MCAM_HIGHER_PRIO) && 2901 ((end - start) != req->ref_entry)) { 2902 start = 0; 2903 end = req->ref_entry; 2904 reverse = true; 2905 goto alloc; 2906 } 2907 } 2908 2909 /* Copy MCAM entry indices into mbox response entry_list. 2910 * Requester always expects indices in ascending order, so 2911 * reverse the list if reverse bitmap is used for allocation. 2912 */ 2913 if (!req->contig && rsp->count) { 2914 index = 0; 2915 for (entry = rsp->count - 1; entry >= 0; entry--) { 2916 if (reverse) 2917 rsp->entry_list[index++] = entry_list[entry]; 2918 else 2919 rsp->entry_list[entry] = entry_list[entry]; 2920 } 2921 } 2922 2923 /* Mark the allocated entries as used and set nixlf mapping */ 2924 for (entry = 0; entry < rsp->count; entry++) { 2925 index = req->contig ? 2926 (rsp->entry + entry) : rsp->entry_list[entry]; 2927 npc_mcam_set_bit(mcam, index); 2928 mcam->entry2pfvf_map[index] = pcifunc; 2929 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 2930 } 2931 2932 /* Update available free count in mbox response */ 2933 rsp->free_count = mcam->bmap_fcnt; 2934 2935 mutex_unlock(&mcam->lock); 2936 return 0; 2937 } 2938 2939 /* Marks bitmaps to reserved the mcam slot */ 2940 void npc_mcam_rsrcs_reserve(struct rvu *rvu, int blkaddr, int entry_idx) 2941 { 2942 struct npc_mcam *mcam = &rvu->hw->mcam; 2943 2944 npc_mcam_set_bit(mcam, entry_idx); 2945 } 2946 2947 int npc_config_cntr_default_entries(struct rvu *rvu, bool enable) 2948 { 2949 struct npc_mcam *mcam = &rvu->hw->mcam; 2950 struct npc_install_flow_rsp rsp; 2951 struct rvu_npc_mcam_rule *rule; 2952 int blkaddr; 2953 2954 /* Counter is set for each rule by default */ 2955 if (is_cn20k(rvu->pdev)) 2956 return -EINVAL; 2957 2958 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2959 if (blkaddr < 0) 2960 return -EINVAL; 2961 2962 mutex_lock(&mcam->lock); 2963 list_for_each_entry(rule, &mcam->mcam_rules, list) { 2964 if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, rule->entry)) 2965 continue; 2966 if (!rule->default_rule) 2967 continue; 2968 if (enable && !rule->has_cntr) { /* Alloc and map new counter */ 2969 __rvu_mcam_add_counter_to_rule(rvu, rule->owner, 2970 rule, &rsp); 2971 if (rsp.counter < 0) { 2972 dev_err(rvu->dev, 2973 "%s: Failed to allocate cntr for default rule (err=%d)\n", 2974 __func__, rsp.counter); 2975 break; 2976 } 2977 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2978 rule->entry, rsp.counter); 2979 /* Reset counter before use */ 2980 rvu_write64(rvu, blkaddr, 2981 NPC_AF_MATCH_STATX(rule->cntr), 0x0); 2982 } 2983 2984 /* Free and unmap counter */ 2985 if (!enable && rule->has_cntr) 2986 __rvu_mcam_remove_counter_from_rule(rvu, rule->owner, 2987 rule); 2988 } 2989 mutex_unlock(&mcam->lock); 2990 2991 return 0; 2992 } 2993 2994 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 2995 struct npc_mcam_alloc_entry_req *req, 2996 struct npc_mcam_alloc_entry_rsp *rsp) 2997 { 2998 struct npc_mcam *mcam = &rvu->hw->mcam; 2999 u16 pcifunc = req->hdr.pcifunc; 3000 int blkaddr; 3001 3002 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3003 if (blkaddr < 0) 3004 return NPC_MCAM_INVALID_REQ; 3005 3006 rsp->entry = NPC_MCAM_ENTRY_INVALID; 3007 rsp->free_count = 0; 3008 3009 /* Check if ref_entry is greater that the range 3010 * then set it to max value. 3011 */ 3012 if (req->ref_entry > mcam->bmap_entries) 3013 req->ref_entry = mcam->bmap_entries; 3014 3015 /* ref_entry can't be '0' if requested priority is high. 3016 * Can't be last entry if requested priority is low. 3017 */ 3018 if ((!req->ref_entry && req->ref_prio == NPC_MCAM_HIGHER_PRIO) || 3019 (req->ref_entry == mcam->bmap_entries && 3020 req->ref_prio == NPC_MCAM_LOWER_PRIO)) 3021 return NPC_MCAM_INVALID_REQ; 3022 3023 /* Since list of allocated indices needs to be sent to requester, 3024 * max number of non-contiguous entries per mbox msg is limited. 3025 */ 3026 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) { 3027 dev_err(rvu->dev, 3028 "%s: %d Non-contiguous MCAM entries requested is more than max (%d) allowed\n", 3029 __func__, req->count, NPC_MAX_NONCONTIG_ENTRIES); 3030 return NPC_MCAM_INVALID_REQ; 3031 } 3032 3033 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 3034 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 3035 return NPC_MCAM_ALLOC_DENIED; 3036 3037 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 3038 } 3039 3040 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 3041 struct npc_mcam_free_entry_req *req, 3042 struct msg_rsp *rsp) 3043 { 3044 struct npc_mcam *mcam = &rvu->hw->mcam; 3045 u16 pcifunc = req->hdr.pcifunc; 3046 int blkaddr, rc = 0; 3047 u16 cntr; 3048 3049 req->entry = npc_cn20k_vidx2idx(req->entry); 3050 3051 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3052 if (blkaddr < 0) 3053 return NPC_MCAM_INVALID_REQ; 3054 3055 /* Free request from PFFUNC with no NIXLF attached, ignore */ 3056 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 3057 return NPC_MCAM_INVALID_REQ; 3058 3059 mutex_lock(&mcam->lock); 3060 3061 if (req->all) 3062 goto free_all; 3063 3064 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3065 if (rc) 3066 goto exit; 3067 3068 mcam->entry2pfvf_map[req->entry] = NPC_MCAM_INVALID_MAP; 3069 mcam->entry2target_pffunc[req->entry] = 0x0; 3070 npc_mcam_clear_bit(mcam, req->entry); 3071 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 3072 3073 /* Update entry2counter mapping */ 3074 cntr = mcam->entry2cntr_map[req->entry]; 3075 if (cntr != NPC_MCAM_INVALID_MAP) 3076 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3077 req->entry, cntr); 3078 3079 if (is_cn20k(rvu->pdev)) { 3080 rc = npc_cn20k_idx_free(rvu, &req->entry, 1); 3081 if (rc) 3082 dev_err(rvu->dev, 3083 "Failed to free index=%u\n", 3084 req->entry); 3085 } 3086 3087 goto exit; 3088 3089 free_all: 3090 /* Free up all entries allocated to requesting PFFUNC */ 3091 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 3092 exit: 3093 mutex_unlock(&mcam->lock); 3094 return rc; 3095 } 3096 3097 int rvu_mbox_handler_npc_mcam_read_entry(struct rvu *rvu, 3098 struct npc_mcam_read_entry_req *req, 3099 struct npc_mcam_read_entry_rsp *rsp) 3100 { 3101 struct npc_mcam *mcam = &rvu->hw->mcam; 3102 u16 pcifunc = req->hdr.pcifunc; 3103 int blkaddr, rc; 3104 3105 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3106 if (blkaddr < 0) 3107 return NPC_MCAM_INVALID_REQ; 3108 3109 mutex_lock(&mcam->lock); 3110 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3111 if (!rc) { 3112 npc_read_mcam_entry(rvu, mcam, blkaddr, req->entry, 3113 &rsp->entry_data, 3114 &rsp->intf, &rsp->enable); 3115 } 3116 3117 mutex_unlock(&mcam->lock); 3118 return rc; 3119 } 3120 3121 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 3122 struct npc_mcam_write_entry_req *req, 3123 struct msg_rsp *rsp) 3124 { 3125 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 3126 struct npc_mcam *mcam = &rvu->hw->mcam; 3127 u16 pcifunc = req->hdr.pcifunc; 3128 int blkaddr, rc; 3129 u8 nix_intf; 3130 3131 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3132 if (blkaddr < 0) 3133 return NPC_MCAM_INVALID_REQ; 3134 3135 mutex_lock(&mcam->lock); 3136 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3137 if (rc) 3138 goto exit; 3139 3140 if (!is_cn20k(rvu->pdev)) { 3141 /* Verify counter in SoCs other than cn20k */ 3142 if (req->set_cntr && 3143 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 3144 rc = NPC_MCAM_INVALID_REQ; 3145 goto exit; 3146 } 3147 } 3148 3149 if (!is_npc_interface_valid(rvu, req->intf)) { 3150 rc = NPC_MCAM_INVALID_REQ; 3151 goto exit; 3152 } 3153 3154 if (is_npc_intf_tx(req->intf)) 3155 nix_intf = pfvf->nix_tx_intf; 3156 else 3157 nix_intf = pfvf->nix_rx_intf; 3158 3159 /* For AF installed rules, the nix_intf should be set to target NIX */ 3160 if (is_pffunc_af(req->hdr.pcifunc)) 3161 nix_intf = req->intf; 3162 3163 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, nix_intf, 3164 &req->entry_data, req->enable_entry); 3165 3166 if (req->set_cntr) 3167 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3168 req->entry, req->cntr); 3169 3170 rc = 0; 3171 exit: 3172 mutex_unlock(&mcam->lock); 3173 return rc; 3174 } 3175 3176 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 3177 struct npc_mcam_ena_dis_entry_req *req, 3178 struct msg_rsp *rsp) 3179 { 3180 struct npc_mcam *mcam = &rvu->hw->mcam; 3181 u16 pcifunc = req->hdr.pcifunc; 3182 int blkaddr, rc; 3183 3184 req->entry = npc_cn20k_vidx2idx(req->entry); 3185 3186 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3187 if (blkaddr < 0) 3188 return NPC_MCAM_INVALID_REQ; 3189 3190 mutex_lock(&mcam->lock); 3191 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3192 mutex_unlock(&mcam->lock); 3193 if (rc) 3194 return rc; 3195 3196 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 3197 3198 return 0; 3199 } 3200 3201 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 3202 struct npc_mcam_ena_dis_entry_req *req, 3203 struct msg_rsp *rsp) 3204 { 3205 struct npc_mcam *mcam = &rvu->hw->mcam; 3206 u16 pcifunc = req->hdr.pcifunc; 3207 int blkaddr, rc; 3208 3209 req->entry = npc_cn20k_vidx2idx(req->entry); 3210 3211 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3212 if (blkaddr < 0) 3213 return NPC_MCAM_INVALID_REQ; 3214 3215 mutex_lock(&mcam->lock); 3216 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3217 mutex_unlock(&mcam->lock); 3218 if (rc) 3219 return rc; 3220 3221 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 3222 3223 return 0; 3224 } 3225 3226 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 3227 struct npc_mcam_shift_entry_req *req, 3228 struct npc_mcam_shift_entry_rsp *rsp) 3229 { 3230 struct npc_mcam *mcam = &rvu->hw->mcam; 3231 u16 pcifunc = req->hdr.pcifunc; 3232 u16 old_entry, new_entry; 3233 int blkaddr, rc = 0; 3234 u16 index, cntr; 3235 3236 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3237 if (blkaddr < 0) 3238 return NPC_MCAM_INVALID_REQ; 3239 3240 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 3241 return NPC_MCAM_INVALID_REQ; 3242 3243 mutex_lock(&mcam->lock); 3244 for (index = 0; index < req->shift_count; index++) { 3245 old_entry = npc_cn20k_vidx2idx(req->curr_entry[index]); 3246 new_entry = npc_cn20k_vidx2idx(req->new_entry[index]); 3247 3248 /* Check if both old and new entries are valid and 3249 * does belong to this PFFUNC or not. 3250 */ 3251 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 3252 if (rc) 3253 break; 3254 3255 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 3256 if (rc) 3257 break; 3258 3259 /* new_entry should not have a counter mapped */ 3260 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 3261 rc = NPC_MCAM_PERM_DENIED; 3262 break; 3263 } 3264 3265 /* Disable the new_entry */ 3266 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 3267 3268 /* Copy rule from old entry to new entry */ 3269 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry); 3270 3271 /* Copy counter mapping, if any */ 3272 cntr = mcam->entry2cntr_map[old_entry]; 3273 if (cntr != NPC_MCAM_INVALID_MAP) { 3274 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3275 old_entry, cntr); 3276 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3277 new_entry, cntr); 3278 } 3279 3280 /* Enable new_entry and disable old_entry */ 3281 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 3282 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 3283 } 3284 3285 /* If shift has failed then report the failed index */ 3286 if (index != req->shift_count) { 3287 rc = NPC_MCAM_PERM_DENIED; 3288 rsp->failed_entry_idx = index; 3289 } 3290 3291 mutex_unlock(&mcam->lock); 3292 return rc; 3293 } 3294 3295 static int __npc_mcam_alloc_counter(struct rvu *rvu, 3296 struct npc_mcam_alloc_counter_req *req, 3297 struct npc_mcam_alloc_counter_rsp *rsp) 3298 { 3299 struct npc_mcam *mcam = &rvu->hw->mcam; 3300 u16 pcifunc = req->hdr.pcifunc; 3301 u16 max_contig, cntr; 3302 int blkaddr, index; 3303 3304 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3305 if (blkaddr < 0) 3306 return NPC_MCAM_INVALID_REQ; 3307 3308 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 3309 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 3310 return NPC_MCAM_INVALID_REQ; 3311 3312 /* Since list of allocated counter IDs needs to be sent to requester, 3313 * max number of non-contiguous counters per mbox msg is limited. 3314 */ 3315 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 3316 return NPC_MCAM_INVALID_REQ; 3317 3318 /* Check if unused counters are available or not */ 3319 if (!rvu_rsrc_free_count(&mcam->counters)) { 3320 return NPC_MCAM_ALLOC_FAILED; 3321 } 3322 3323 rsp->count = 0; 3324 3325 if (req->contig) { 3326 /* Allocate requested number of contiguous counters, if 3327 * unsuccessful find max contiguous entries available. 3328 */ 3329 index = npc_mcam_find_zero_area(mcam->counters.bmap, 3330 mcam->counters.max, 0, 3331 req->count, &max_contig); 3332 rsp->count = max_contig; 3333 rsp->cntr = index; 3334 for (cntr = index; cntr < (index + max_contig); cntr++) { 3335 __set_bit(cntr, mcam->counters.bmap); 3336 mcam->cntr2pfvf_map[cntr] = pcifunc; 3337 } 3338 } else { 3339 /* Allocate requested number of non-contiguous counters, 3340 * if unsuccessful allocate as many as possible. 3341 */ 3342 for (cntr = 0; cntr < req->count; cntr++) { 3343 index = rvu_alloc_rsrc(&mcam->counters); 3344 if (index < 0) 3345 break; 3346 rsp->cntr_list[cntr] = index; 3347 rsp->count++; 3348 mcam->cntr2pfvf_map[index] = pcifunc; 3349 } 3350 } 3351 3352 return 0; 3353 } 3354 3355 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 3356 struct npc_mcam_alloc_counter_req *req, 3357 struct npc_mcam_alloc_counter_rsp *rsp) 3358 { 3359 struct npc_mcam *mcam = &rvu->hw->mcam; 3360 int err; 3361 3362 /* Counter is not supported for CN20K */ 3363 if (is_cn20k(rvu->pdev)) 3364 return NPC_MCAM_INVALID_REQ; 3365 3366 mutex_lock(&mcam->lock); 3367 3368 err = __npc_mcam_alloc_counter(rvu, req, rsp); 3369 3370 mutex_unlock(&mcam->lock); 3371 return err; 3372 } 3373 3374 static int __npc_mcam_free_counter(struct rvu *rvu, 3375 struct npc_mcam_oper_counter_req *req, 3376 struct msg_rsp *rsp) 3377 { 3378 struct npc_mcam *mcam = &rvu->hw->mcam; 3379 u16 index, entry = 0; 3380 int blkaddr, err; 3381 3382 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3383 if (blkaddr < 0) 3384 return NPC_MCAM_INVALID_REQ; 3385 3386 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3387 if (err) { 3388 return err; 3389 } 3390 3391 /* Mark counter as free/unused */ 3392 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 3393 rvu_free_rsrc(&mcam->counters, req->cntr); 3394 3395 /* Disable all MCAM entry's stats which are using this counter */ 3396 while (entry < mcam->bmap_entries) { 3397 if (!mcam->cntr_refcnt[req->cntr]) 3398 break; 3399 3400 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 3401 if (index >= mcam->bmap_entries) 3402 break; 3403 entry = index + 1; 3404 if (mcam->entry2cntr_map[index] != req->cntr) 3405 continue; 3406 3407 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3408 index, req->cntr); 3409 } 3410 3411 return 0; 3412 } 3413 3414 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 3415 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 3416 { 3417 struct npc_mcam *mcam = &rvu->hw->mcam; 3418 int err; 3419 3420 /* Counter is not supported for CN20K */ 3421 if (is_cn20k(rvu->pdev)) 3422 return NPC_MCAM_INVALID_REQ; 3423 3424 mutex_lock(&mcam->lock); 3425 3426 err = __npc_mcam_free_counter(rvu, req, rsp); 3427 3428 mutex_unlock(&mcam->lock); 3429 3430 return err; 3431 } 3432 3433 void __rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc, 3434 struct rvu_npc_mcam_rule *rule) 3435 { 3436 struct npc_mcam_oper_counter_req free_req = { 0 }; 3437 struct msg_rsp free_rsp; 3438 3439 if (!rule->has_cntr) 3440 return; 3441 3442 free_req.hdr.pcifunc = pcifunc; 3443 free_req.cntr = rule->cntr; 3444 3445 __npc_mcam_free_counter(rvu, &free_req, &free_rsp); 3446 rule->has_cntr = false; 3447 } 3448 3449 void __rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc, 3450 struct rvu_npc_mcam_rule *rule, 3451 struct npc_install_flow_rsp *rsp) 3452 { 3453 struct npc_mcam_alloc_counter_req cntr_req = { 0 }; 3454 struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; 3455 int err; 3456 3457 cntr_req.hdr.pcifunc = pcifunc; 3458 cntr_req.contig = true; 3459 cntr_req.count = 1; 3460 3461 /* we try to allocate a counter to track the stats of this 3462 * rule. If counter could not be allocated then proceed 3463 * without counter because counters are limited than entries. 3464 */ 3465 err = __npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 3466 if (!err && cntr_rsp.count) { 3467 rule->cntr = cntr_rsp.cntr; 3468 rule->has_cntr = true; 3469 rsp->counter = rule->cntr; 3470 } else { 3471 rsp->counter = err; 3472 } 3473 } 3474 3475 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 3476 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 3477 { 3478 struct npc_mcam *mcam = &rvu->hw->mcam; 3479 u16 index, entry = 0; 3480 int blkaddr, rc; 3481 3482 /* Counter is not supported for CN20K */ 3483 if (is_cn20k(rvu->pdev)) 3484 return NPC_MCAM_INVALID_REQ; 3485 3486 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3487 if (blkaddr < 0) 3488 return NPC_MCAM_INVALID_REQ; 3489 3490 mutex_lock(&mcam->lock); 3491 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3492 if (rc) 3493 goto exit; 3494 3495 /* Unmap the MCAM entry and counter */ 3496 if (!req->all) { 3497 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 3498 if (rc) 3499 goto exit; 3500 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3501 req->entry, req->cntr); 3502 goto exit; 3503 } 3504 3505 /* Disable all MCAM entry's stats which are using this counter */ 3506 while (entry < mcam->bmap_entries) { 3507 if (!mcam->cntr_refcnt[req->cntr]) 3508 break; 3509 3510 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 3511 if (index >= mcam->bmap_entries) 3512 break; 3513 entry = index + 1; 3514 3515 if (mcam->entry2cntr_map[index] != req->cntr) 3516 continue; 3517 3518 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3519 index, req->cntr); 3520 } 3521 exit: 3522 mutex_unlock(&mcam->lock); 3523 return rc; 3524 } 3525 3526 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 3527 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 3528 { 3529 struct npc_mcam *mcam = &rvu->hw->mcam; 3530 int blkaddr, err, index, bank; 3531 3532 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3533 if (blkaddr < 0) 3534 return NPC_MCAM_INVALID_REQ; 3535 3536 /* For cn20k, npc mcam index is passed as cntr, as each 3537 * mcam entry has corresponding counter. 3538 */ 3539 if (is_cn20k(rvu->pdev)) { 3540 index = req->cntr & (mcam->banksize - 1); 3541 bank = npc_get_bank(mcam, req->cntr); 3542 rvu_write64(rvu, blkaddr, 3543 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0); 3544 return 0; 3545 } 3546 3547 mutex_lock(&mcam->lock); 3548 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3549 mutex_unlock(&mcam->lock); 3550 if (err) 3551 return err; 3552 3553 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 3554 3555 return 0; 3556 } 3557 3558 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 3559 struct npc_mcam_oper_counter_req *req, 3560 struct npc_mcam_oper_counter_rsp *rsp) 3561 { 3562 struct npc_mcam *mcam = &rvu->hw->mcam; 3563 int blkaddr, err, index, bank; 3564 u64 regval; 3565 3566 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3567 if (blkaddr < 0) 3568 return NPC_MCAM_INVALID_REQ; 3569 3570 /* In CN20K, mcam index is passed cntr. Each cn20k mcam entry 3571 * has its own counter. No need to verify the counter index. 3572 */ 3573 if (is_cn20k(rvu->pdev)) { 3574 index = req->cntr & (mcam->banksize - 1); 3575 bank = npc_get_bank(mcam, req->cntr); 3576 regval = rvu_read64(rvu, blkaddr, 3577 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, 3578 bank)); 3579 rsp->stat = regval; 3580 return 0; 3581 } 3582 3583 mutex_lock(&mcam->lock); 3584 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3585 mutex_unlock(&mcam->lock); 3586 if (err) 3587 return err; 3588 3589 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 3590 rsp->stat &= BIT_ULL(48) - 1; 3591 3592 return 0; 3593 } 3594 3595 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 3596 struct npc_mcam_alloc_and_write_entry_req *req, 3597 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 3598 { 3599 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 3600 struct npc_mcam_alloc_counter_req cntr_req; 3601 struct npc_mcam_alloc_counter_rsp cntr_rsp; 3602 struct npc_mcam_alloc_entry_req entry_req; 3603 struct npc_mcam_alloc_entry_rsp entry_rsp; 3604 struct npc_mcam *mcam = &rvu->hw->mcam; 3605 u16 entry = NPC_MCAM_ENTRY_INVALID; 3606 u16 cntr = NPC_MCAM_ENTRY_INVALID; 3607 int blkaddr, rc; 3608 u8 nix_intf; 3609 3610 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3611 if (blkaddr < 0) 3612 return NPC_MCAM_INVALID_REQ; 3613 3614 if (!is_npc_interface_valid(rvu, req->intf)) 3615 return NPC_MCAM_INVALID_REQ; 3616 3617 /* Try to allocate a MCAM entry */ 3618 entry_req.hdr.pcifunc = req->hdr.pcifunc; 3619 entry_req.contig = true; 3620 entry_req.ref_prio = req->ref_prio; 3621 entry_req.ref_entry = req->ref_entry; 3622 entry_req.count = 1; 3623 3624 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 3625 &entry_req, &entry_rsp); 3626 if (rc) 3627 return rc; 3628 3629 if (!entry_rsp.count) 3630 return NPC_MCAM_ALLOC_FAILED; 3631 3632 entry = entry_rsp.entry; 3633 3634 if (!req->alloc_cntr) 3635 goto write_entry; 3636 3637 /* Now allocate counter */ 3638 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 3639 cntr_req.contig = true; 3640 cntr_req.count = 1; 3641 3642 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 3643 if (rc) { 3644 /* Free allocated MCAM entry */ 3645 mutex_lock(&mcam->lock); 3646 mcam->entry2pfvf_map[entry] = NPC_MCAM_INVALID_MAP; 3647 npc_mcam_clear_bit(mcam, entry); 3648 mutex_unlock(&mcam->lock); 3649 return rc; 3650 } 3651 3652 cntr = cntr_rsp.cntr; 3653 3654 write_entry: 3655 mutex_lock(&mcam->lock); 3656 3657 if (is_npc_intf_tx(req->intf)) 3658 nix_intf = pfvf->nix_tx_intf; 3659 else 3660 nix_intf = pfvf->nix_rx_intf; 3661 3662 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, nix_intf, 3663 &req->entry_data, req->enable_entry); 3664 3665 if (req->alloc_cntr) 3666 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 3667 mutex_unlock(&mcam->lock); 3668 3669 rsp->entry = entry; 3670 rsp->cntr = cntr; 3671 3672 return 0; 3673 } 3674 3675 #define GET_KEX_CFG(intf) \ 3676 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 3677 3678 #define GET_KEX_FLAGS(ld) \ 3679 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 3680 3681 #define GET_KEX_LD(intf, lid, lt, ld) \ 3682 rvu_read64(rvu, BLKADDR_NPC, \ 3683 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 3684 3685 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 3686 rvu_read64(rvu, BLKADDR_NPC, \ 3687 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 3688 3689 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 3690 struct npc_get_kex_cfg_rsp *rsp) 3691 { 3692 int lid, lt, ld, fl; 3693 3694 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 3695 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 3696 for (lid = 0; lid < NPC_MAX_LID; lid++) { 3697 for (lt = 0; lt < NPC_MAX_LT; lt++) { 3698 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3699 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 3700 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 3701 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 3702 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 3703 } 3704 } 3705 } 3706 for (ld = 0; ld < NPC_MAX_LD; ld++) 3707 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 3708 3709 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3710 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 3711 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 3712 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 3713 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 3714 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 3715 } 3716 } 3717 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 3718 return 0; 3719 } 3720 3721 static int 3722 npc_set_var_len_offset_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind, 3723 u8 var_len_off, u8 var_len_off_mask, u8 shift_dir) 3724 { 3725 struct npc_kpu_action0 *act0; 3726 u8 shift_count = 0; 3727 int blkaddr; 3728 u64 val; 3729 3730 if (!var_len_off_mask) 3731 return -EINVAL; 3732 3733 if (var_len_off_mask != 0xff) { 3734 if (shift_dir) 3735 shift_count = __ffs(var_len_off_mask); 3736 else 3737 shift_count = (8 - __fls(var_len_off_mask)); 3738 } 3739 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 3740 if (blkaddr < 0) { 3741 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 3742 return -EINVAL; 3743 } 3744 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 3745 act0 = (struct npc_kpu_action0 *)&val; 3746 act0->var_len_shift = shift_count; 3747 act0->var_len_right = shift_dir; 3748 act0->var_len_mask = var_len_off_mask; 3749 act0->var_len_offset = var_len_off; 3750 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 3751 return 0; 3752 } 3753 3754 int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir, 3755 u64 pkind, u8 var_len_off, u8 var_len_off_mask, 3756 u8 shift_dir) 3757 3758 { 3759 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 3760 int blkaddr, nixlf, rc, intf_mode; 3761 int pf = rvu_get_pf(rvu->pdev, pcifunc); 3762 u64 rxpkind, txpkind; 3763 u8 cgx_id, lmac_id; 3764 3765 /* use default pkind to disable edsa/higig */ 3766 rxpkind = rvu_npc_get_pkind(rvu, pf); 3767 txpkind = NPC_TX_DEF_PKIND; 3768 intf_mode = NPC_INTF_MODE_DEF; 3769 3770 if (mode & OTX2_PRIV_FLAGS_CUSTOM) { 3771 if (pkind == NPC_RX_CUSTOM_PRE_L2_PKIND) { 3772 rc = npc_set_var_len_offset_pkind(rvu, pcifunc, pkind, 3773 var_len_off, 3774 var_len_off_mask, 3775 shift_dir); 3776 if (rc) 3777 return rc; 3778 } 3779 rxpkind = pkind; 3780 txpkind = pkind; 3781 } 3782 3783 if (dir & PKIND_RX) { 3784 /* rx pkind set req valid only for cgx mapped PFs */ 3785 if (!is_cgx_config_permitted(rvu, pcifunc)) 3786 return 0; 3787 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 3788 3789 rc = cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, 3790 rxpkind); 3791 if (rc) 3792 return rc; 3793 } 3794 3795 if (dir & PKIND_TX) { 3796 /* Tx pkind set request valid if PCIFUNC has NIXLF attached */ 3797 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 3798 if (rc) 3799 return rc; 3800 3801 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), 3802 txpkind); 3803 } 3804 3805 pfvf->intf_mode = intf_mode; 3806 return 0; 3807 } 3808 3809 int rvu_mbox_handler_npc_set_pkind(struct rvu *rvu, struct npc_set_pkind *req, 3810 struct msg_rsp *rsp) 3811 { 3812 return rvu_npc_set_parse_mode(rvu, req->hdr.pcifunc, req->mode, 3813 req->dir, req->pkind, req->var_len_off, 3814 req->var_len_off_mask, req->shift_dir); 3815 } 3816 3817 int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu, 3818 struct msg_req *req, 3819 struct npc_mcam_read_base_rule_rsp *rsp) 3820 { 3821 struct npc_mcam *mcam = &rvu->hw->mcam; 3822 int index, blkaddr, nixlf, rc = 0; 3823 u16 pcifunc = req->hdr.pcifunc; 3824 struct rvu_pfvf *pfvf; 3825 u8 intf, enable; 3826 3827 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3828 if (blkaddr < 0) 3829 return NPC_MCAM_INVALID_REQ; 3830 3831 /* Return the channel number in case of PF */ 3832 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 3833 pfvf = rvu_get_pfvf(rvu, pcifunc); 3834 rsp->entry.kw[0] = pfvf->rx_chan_base; 3835 rsp->entry.kw_mask[0] = 0xFFFULL; 3836 goto out; 3837 } 3838 3839 /* Find the pkt steering rule installed by PF to this VF */ 3840 mutex_lock(&mcam->lock); 3841 for (index = 0; index < mcam->bmap_entries; index++) { 3842 if (mcam->entry2target_pffunc[index] == pcifunc) 3843 goto read_entry; 3844 } 3845 3846 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 3847 if (rc < 0) { 3848 mutex_unlock(&mcam->lock); 3849 goto out; 3850 } 3851 /* Read the default ucast entry if there is no pkt steering rule */ 3852 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 3853 NIXLF_UCAST_ENTRY); 3854 read_entry: 3855 /* Read the mcam entry */ 3856 npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf, 3857 &enable); 3858 mutex_unlock(&mcam->lock); 3859 out: 3860 return rc; 3861 } 3862 3863 int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu, 3864 struct npc_mcam_get_stats_req *req, 3865 struct npc_mcam_get_stats_rsp *rsp) 3866 { 3867 struct npc_mcam *mcam = &rvu->hw->mcam; 3868 u16 index, cntr; 3869 int blkaddr; 3870 u64 regval; 3871 u32 bank; 3872 3873 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3874 if (blkaddr < 0) 3875 return NPC_MCAM_INVALID_REQ; 3876 3877 req->entry = npc_cn20k_vidx2idx(req->entry); 3878 3879 index = req->entry & (mcam->banksize - 1); 3880 bank = npc_get_bank(mcam, req->entry); 3881 3882 mutex_lock(&mcam->lock); 3883 3884 if (is_cn20k(rvu->pdev)) { 3885 regval = rvu_read64(rvu, blkaddr, 3886 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, 3887 bank)); 3888 rsp->stat_ena = 1; 3889 rsp->stat = regval; 3890 mutex_unlock(&mcam->lock); 3891 return 0; 3892 } 3893 3894 /* read MCAM entry STAT_ACT register */ 3895 regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, 3896 bank)); 3897 3898 if (!(regval & rvu->hw->npc_stat_ena)) { 3899 rsp->stat_ena = 0; 3900 mutex_unlock(&mcam->lock); 3901 return 0; 3902 } 3903 3904 cntr = regval & 0x1FF; 3905 3906 rsp->stat_ena = 1; 3907 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(cntr)); 3908 rsp->stat &= BIT_ULL(48) - 1; 3909 3910 mutex_unlock(&mcam->lock); 3911 3912 return 0; 3913 } 3914 3915 void rvu_npc_clear_ucast_entry(struct rvu *rvu, int pcifunc, int nixlf) 3916 { 3917 struct npc_mcam *mcam = &rvu->hw->mcam; 3918 struct rvu_npc_mcam_rule *rule; 3919 int ucast_idx, blkaddr; 3920 3921 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3922 if (blkaddr < 0) 3923 return; 3924 3925 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 3926 nixlf, NIXLF_UCAST_ENTRY); 3927 3928 npc_enable_mcam_entry(rvu, mcam, blkaddr, ucast_idx, false); 3929 3930 npc_set_mcam_action(rvu, mcam, blkaddr, ucast_idx, 0); 3931 3932 npc_clear_mcam_entry(rvu, mcam, blkaddr, ucast_idx); 3933 3934 mutex_lock(&mcam->lock); 3935 list_for_each_entry(rule, &mcam->mcam_rules, list) { 3936 if (rule->entry == ucast_idx) { 3937 list_del(&rule->list); 3938 kfree(rule); 3939 break; 3940 } 3941 } 3942 mutex_unlock(&mcam->lock); 3943 } 3944