1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Ethernet driver 3 * 4 * Copyright (C) 2021 Marvell. 5 * 6 */ 7 8 #include "cn10k.h" 9 #include "otx2_reg.h" 10 #include "otx2_struct.h" 11 12 static struct dev_hw_ops otx2_hw_ops = { 13 .sq_aq_init = otx2_sq_aq_init, 14 .sqe_flush = otx2_sqe_flush, 15 .aura_freeptr = otx2_aura_freeptr, 16 .refill_pool_ptrs = otx2_refill_pool_ptrs, 17 }; 18 19 static struct dev_hw_ops cn10k_hw_ops = { 20 .sq_aq_init = cn10k_sq_aq_init, 21 .sqe_flush = cn10k_sqe_flush, 22 .aura_freeptr = cn10k_aura_freeptr, 23 .refill_pool_ptrs = cn10k_refill_pool_ptrs, 24 }; 25 26 int cn10k_lmtst_init(struct otx2_nic *pfvf) 27 { 28 29 struct lmtst_tbl_setup_req *req; 30 struct otx2_lmt_info *lmt_info; 31 int err, cpu; 32 33 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { 34 pfvf->hw_ops = &otx2_hw_ops; 35 return 0; 36 } 37 38 pfvf->hw_ops = &cn10k_hw_ops; 39 /* Total LMTLINES = num_online_cpus() * 32 (For Burst flush).*/ 40 pfvf->tot_lmt_lines = (num_online_cpus() * LMT_BURST_SIZE); 41 pfvf->hw.lmt_info = alloc_percpu(struct otx2_lmt_info); 42 43 mutex_lock(&pfvf->mbox.lock); 44 req = otx2_mbox_alloc_msg_lmtst_tbl_setup(&pfvf->mbox); 45 if (!req) { 46 mutex_unlock(&pfvf->mbox.lock); 47 return -ENOMEM; 48 } 49 50 req->use_local_lmt_region = true; 51 52 err = qmem_alloc(pfvf->dev, &pfvf->dync_lmt, pfvf->tot_lmt_lines, 53 LMT_LINE_SIZE); 54 if (err) { 55 mutex_unlock(&pfvf->mbox.lock); 56 return err; 57 } 58 pfvf->hw.lmt_base = (u64 *)pfvf->dync_lmt->base; 59 req->lmt_iova = (u64)pfvf->dync_lmt->iova; 60 61 err = otx2_sync_mbox_msg(&pfvf->mbox); 62 mutex_unlock(&pfvf->mbox.lock); 63 64 for_each_possible_cpu(cpu) { 65 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, cpu); 66 lmt_info->lmt_addr = ((u64)pfvf->hw.lmt_base + 67 (cpu * LMT_BURST_SIZE * LMT_LINE_SIZE)); 68 lmt_info->lmt_id = cpu * LMT_BURST_SIZE; 69 } 70 71 return 0; 72 } 73 EXPORT_SYMBOL(cn10k_lmtst_init); 74 75 int cn10k_sq_aq_init(void *dev, u16 qidx, u8 chan_offset, u16 sqb_aura) 76 { 77 struct nix_cn10k_aq_enq_req *aq; 78 struct otx2_nic *pfvf = dev; 79 80 /* Get memory to put this msg */ 81 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 82 if (!aq) 83 return -ENOMEM; 84 85 aq->sq.cq = pfvf->hw.rx_queues + qidx; 86 aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ 87 aq->sq.cq_ena = 1; 88 aq->sq.ena = 1; 89 aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); 90 aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); 91 aq->sq.default_chan = pfvf->hw.tx_chan_base + chan_offset; 92 aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ 93 aq->sq.sqb_aura = sqb_aura; 94 aq->sq.sq_int_ena = NIX_SQINT_BITS; 95 aq->sq.qint_idx = 0; 96 /* Due pipelining impact minimum 2000 unused SQ CQE's 97 * need to maintain to avoid CQ overflow. 98 */ 99 aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt)); 100 101 /* Fill AQ info */ 102 aq->qidx = qidx; 103 aq->ctype = NIX_AQ_CTYPE_SQ; 104 aq->op = NIX_AQ_INSTOP_INIT; 105 106 return otx2_sync_mbox_msg(&pfvf->mbox); 107 } 108 109 #define NPA_MAX_BURST 16 110 int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) 111 { 112 struct otx2_nic *pfvf = dev; 113 int cnt = cq->pool_ptrs; 114 u64 ptrs[NPA_MAX_BURST]; 115 struct otx2_pool *pool; 116 dma_addr_t bufptr; 117 int num_ptrs = 1; 118 119 pool = &pfvf->qset.pool[cq->cq_idx]; 120 121 /* Refill pool with new buffers */ 122 while (cq->pool_ptrs) { 123 if (otx2_alloc_buffer(pfvf, cq, &bufptr)) { 124 if (num_ptrs--) 125 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, 126 num_ptrs); 127 break; 128 } 129 cq->pool_ptrs--; 130 ptrs[num_ptrs] = pool->xsk_pool ? 131 (u64)bufptr : (u64)bufptr + OTX2_HEAD_ROOM; 132 133 num_ptrs++; 134 if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) { 135 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, 136 num_ptrs); 137 num_ptrs = 1; 138 } 139 } 140 return cnt - cq->pool_ptrs; 141 } 142 143 void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx) 144 { 145 struct otx2_lmt_info *lmt_info; 146 struct otx2_nic *pfvf = dev; 147 u64 val = 0, tar_addr = 0; 148 149 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id()); 150 /* FIXME: val[0:10] LMT_ID. 151 * [12:15] no of LMTST - 1 in the burst. 152 * [19:63] data size of each LMTST in the burst except first. 153 */ 154 val = (lmt_info->lmt_id & 0x7FF); 155 /* Target address for LMTST flush tells HW how many 128bit 156 * words are present. 157 * tar_addr[6:4] size of first LMTST - 1 in units of 128b. 158 */ 159 tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4; 160 dma_wmb(); 161 memcpy((u64 *)lmt_info->lmt_addr, sq->sqe_base, size); 162 cn10k_lmt_flush(val, tar_addr); 163 164 sq->head++; 165 sq->head &= (sq->sqe_cnt - 1); 166 } 167 168 int cn10k_free_all_ipolicers(struct otx2_nic *pfvf) 169 { 170 struct nix_bandprof_free_req *req; 171 int rc; 172 173 if (is_dev_otx2(pfvf->pdev)) 174 return 0; 175 176 mutex_lock(&pfvf->mbox.lock); 177 178 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox); 179 if (!req) { 180 rc = -ENOMEM; 181 goto out; 182 } 183 184 /* Free all bandwidth profiles allocated */ 185 req->free_all = true; 186 187 rc = otx2_sync_mbox_msg(&pfvf->mbox); 188 out: 189 mutex_unlock(&pfvf->mbox.lock); 190 return rc; 191 } 192 193 int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf) 194 { 195 struct nix_bandprof_alloc_req *req; 196 struct nix_bandprof_alloc_rsp *rsp; 197 int rc; 198 199 req = otx2_mbox_alloc_msg_nix_bandprof_alloc(&pfvf->mbox); 200 if (!req) 201 return -ENOMEM; 202 203 req->prof_count[BAND_PROF_LEAF_LAYER] = 1; 204 205 rc = otx2_sync_mbox_msg(&pfvf->mbox); 206 if (rc) 207 goto out; 208 209 rsp = (struct nix_bandprof_alloc_rsp *) 210 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); 211 if (IS_ERR(rsp)) { 212 rc = PTR_ERR(rsp); 213 goto out; 214 } 215 216 if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) { 217 rc = -EIO; 218 goto out; 219 } 220 221 *leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0]; 222 out: 223 if (rc) { 224 dev_warn(pfvf->dev, 225 "Failed to allocate ingress bandwidth policer\n"); 226 } 227 228 return rc; 229 } 230 231 int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf) 232 { 233 struct otx2_hw *hw = &pfvf->hw; 234 int ret; 235 236 mutex_lock(&pfvf->mbox.lock); 237 238 ret = cn10k_alloc_leaf_profile(pfvf, &hw->matchall_ipolicer); 239 240 mutex_unlock(&pfvf->mbox.lock); 241 242 return ret; 243 } 244 245 #define POLICER_TIMESTAMP 1 /* 1 second */ 246 #define MAX_RATE_EXP 22 /* Valid rate exponent range: 0 - 22 */ 247 248 static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp, 249 u32 *burst_mantissa) 250 { 251 int tmp; 252 253 /* Burst is calculated as 254 * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT] 255 * This is the upper limit on number tokens (bytes) that 256 * can be accumulated in the bucket. 257 */ 258 *burst_exp = ilog2(burst); 259 if (burst < 256) { 260 /* No float: can't express mantissa in this case */ 261 *burst_mantissa = 0; 262 return; 263 } 264 265 if (*burst_exp > MAX_RATE_EXP) 266 *burst_exp = MAX_RATE_EXP; 267 268 /* Calculate mantissa 269 * Find remaining bytes 'burst - 2^burst_exp' 270 * mantissa = (remaining bytes) / 2^ (burst_exp - 8) 271 */ 272 tmp = burst - rounddown_pow_of_two(burst); 273 *burst_mantissa = tmp / (1UL << (*burst_exp - 8)); 274 } 275 276 static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp, 277 u32 *rate_mantissa, u32 *rdiv) 278 { 279 u32 div = 0; 280 u32 exp = 0; 281 u64 tmp; 282 283 /* Figure out mantissa, exponent and divider from given max pkt rate 284 * 285 * To achieve desired rate HW adds 286 * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every 287 * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket. 288 * Here policer timeunit is 2 usecs and rate is in bits per sec. 289 * Since floating point cannot be used below algorithm uses 1000000 290 * scale factor to support rates upto 100Gbps. 291 */ 292 tmp = rate * 32 * 2; 293 if (tmp < 256000000) { 294 while (tmp < 256000000) { 295 tmp = tmp * 2; 296 div++; 297 } 298 } else { 299 for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++) 300 tmp = tmp / 2; 301 302 if (exp > MAX_RATE_EXP) 303 exp = MAX_RATE_EXP; 304 } 305 306 *rate_mantissa = (tmp - 256000000) / 1000000; 307 *rate_exp = exp; 308 *rdiv = div; 309 } 310 311 int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx, 312 u16 policer, bool map) 313 { 314 struct nix_cn10k_aq_enq_req *aq; 315 316 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 317 if (!aq) 318 return -ENOMEM; 319 320 /* Enable policing and set the bandwidth profile (policer) index */ 321 if (map) 322 aq->rq.policer_ena = 1; 323 else 324 aq->rq.policer_ena = 0; 325 aq->rq_mask.policer_ena = 1; 326 327 aq->rq.band_prof_id = policer; 328 aq->rq_mask.band_prof_id = GENMASK(9, 0); 329 330 /* Fill AQ info */ 331 aq->qidx = rq_idx; 332 aq->ctype = NIX_AQ_CTYPE_RQ; 333 aq->op = NIX_AQ_INSTOP_WRITE; 334 335 return otx2_sync_mbox_msg(&pfvf->mbox); 336 } 337 338 int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf) 339 { 340 struct nix_bandprof_free_req *req; 341 342 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox); 343 if (!req) 344 return -ENOMEM; 345 346 req->prof_count[BAND_PROF_LEAF_LAYER] = 1; 347 req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf; 348 349 return otx2_sync_mbox_msg(&pfvf->mbox); 350 } 351 352 int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf) 353 { 354 struct otx2_hw *hw = &pfvf->hw; 355 int qidx, rc; 356 357 mutex_lock(&pfvf->mbox.lock); 358 359 /* Remove RQ's policer mapping */ 360 for (qidx = 0; qidx < hw->rx_queues; qidx++) 361 cn10k_map_unmap_rq_policer(pfvf, qidx, 362 hw->matchall_ipolicer, false); 363 364 rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer); 365 366 mutex_unlock(&pfvf->mbox.lock); 367 return rc; 368 } 369 370 int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile, 371 u32 burst, u64 rate, bool pps) 372 { 373 struct nix_cn10k_aq_enq_req *aq; 374 u32 burst_exp, burst_mantissa; 375 u32 rate_exp, rate_mantissa; 376 u32 rdiv; 377 378 /* Get exponent and mantissa values for the desired rate */ 379 cn10k_get_ingress_burst_cfg(burst, &burst_exp, &burst_mantissa); 380 cn10k_get_ingress_rate_cfg(rate, &rate_exp, &rate_mantissa, &rdiv); 381 382 /* Init bandwidth profile */ 383 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 384 if (!aq) 385 return -ENOMEM; 386 387 /* Set initial color mode to blind */ 388 aq->prof.icolor = 0x03; 389 aq->prof_mask.icolor = 0x03; 390 391 /* Set rate and burst values */ 392 aq->prof.cir_exponent = rate_exp; 393 aq->prof_mask.cir_exponent = 0x1F; 394 395 aq->prof.cir_mantissa = rate_mantissa; 396 aq->prof_mask.cir_mantissa = 0xFF; 397 398 aq->prof.cbs_exponent = burst_exp; 399 aq->prof_mask.cbs_exponent = 0x1F; 400 401 aq->prof.cbs_mantissa = burst_mantissa; 402 aq->prof_mask.cbs_mantissa = 0xFF; 403 404 aq->prof.rdiv = rdiv; 405 aq->prof_mask.rdiv = 0xF; 406 407 if (pps) { 408 /* The amount of decremented tokens is calculated according to 409 * the following equation: 410 * max([ LMODE ? 0 : (packet_length - LXPTR)] + 411 * ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT], 412 * 1/256) 413 * if LMODE is 1 then rate limiting will be based on 414 * PPS otherwise bps. 415 * The aim of the ADJUST value is to specify a token cost per 416 * packet in contrary to the packet length that specifies a 417 * cost per byte. To rate limit based on PPS adjust mantissa 418 * is set as 384 and exponent as 1 so that number of tokens 419 * decremented becomes 1 i.e, 1 token per packeet. 420 */ 421 aq->prof.adjust_exponent = 1; 422 aq->prof_mask.adjust_exponent = 0x1F; 423 424 aq->prof.adjust_mantissa = 384; 425 aq->prof_mask.adjust_mantissa = 0x1FF; 426 427 aq->prof.lmode = 0x1; 428 aq->prof_mask.lmode = 0x1; 429 } 430 431 /* Two rate three color marker 432 * With PEIR/EIR set to zero, color will be either green or red 433 */ 434 aq->prof.meter_algo = 2; 435 aq->prof_mask.meter_algo = 0x3; 436 437 aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP; 438 aq->prof_mask.rc_action = 0x3; 439 440 aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; 441 aq->prof_mask.yc_action = 0x3; 442 443 aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; 444 aq->prof_mask.gc_action = 0x3; 445 446 /* Setting exponent value as 24 and mantissa as 0 configures 447 * the bucket with zero values making bucket unused. Peak 448 * information rate and Excess information rate buckets are 449 * unused here. 450 */ 451 aq->prof.peir_exponent = 24; 452 aq->prof_mask.peir_exponent = 0x1F; 453 454 aq->prof.peir_mantissa = 0; 455 aq->prof_mask.peir_mantissa = 0xFF; 456 457 aq->prof.pebs_exponent = 24; 458 aq->prof_mask.pebs_exponent = 0x1F; 459 460 aq->prof.pebs_mantissa = 0; 461 aq->prof_mask.pebs_mantissa = 0xFF; 462 463 aq->prof.hl_en = 0; 464 aq->prof_mask.hl_en = 1; 465 466 /* Fill AQ info */ 467 aq->qidx = profile; 468 aq->ctype = NIX_AQ_CTYPE_BANDPROF; 469 aq->op = NIX_AQ_INSTOP_WRITE; 470 471 return otx2_sync_mbox_msg(&pfvf->mbox); 472 } 473 474 int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf, 475 u32 burst, u64 rate) 476 { 477 struct otx2_hw *hw = &pfvf->hw; 478 int qidx, rc; 479 480 mutex_lock(&pfvf->mbox.lock); 481 482 rc = cn10k_set_ipolicer_rate(pfvf, hw->matchall_ipolicer, burst, 483 rate, false); 484 if (rc) 485 goto out; 486 487 for (qidx = 0; qidx < hw->rx_queues; qidx++) { 488 rc = cn10k_map_unmap_rq_policer(pfvf, qidx, 489 hw->matchall_ipolicer, true); 490 if (rc) 491 break; 492 } 493 494 out: 495 mutex_unlock(&pfvf->mbox.lock); 496 return rc; 497 } 498