1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Ethernet driver
3 *
4 * Copyright (C) 2021 Marvell.
5 *
6 */
7
8 #include "cn10k.h"
9 #include "otx2_reg.h"
10 #include "otx2_struct.h"
11
12 static struct dev_hw_ops otx2_hw_ops = {
13 .sq_aq_init = otx2_sq_aq_init,
14 .sqe_flush = otx2_sqe_flush,
15 .aura_freeptr = otx2_aura_freeptr,
16 .refill_pool_ptrs = otx2_refill_pool_ptrs,
17 .pfaf_mbox_intr_handler = otx2_pfaf_mbox_intr_handler,
18 .aura_aq_init = otx2_aura_aq_init,
19 .pool_aq_init = otx2_pool_aq_init,
20 };
21
22 static struct dev_hw_ops cn10k_hw_ops = {
23 .sq_aq_init = cn10k_sq_aq_init,
24 .sqe_flush = cn10k_sqe_flush,
25 .aura_freeptr = cn10k_aura_freeptr,
26 .refill_pool_ptrs = cn10k_refill_pool_ptrs,
27 .pfaf_mbox_intr_handler = otx2_pfaf_mbox_intr_handler,
28 .aura_aq_init = otx2_aura_aq_init,
29 .pool_aq_init = otx2_pool_aq_init,
30 };
31
otx2_init_hw_ops(struct otx2_nic * pfvf)32 void otx2_init_hw_ops(struct otx2_nic *pfvf)
33 {
34 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) {
35 pfvf->hw_ops = &otx2_hw_ops;
36 return;
37 }
38
39 pfvf->hw_ops = &cn10k_hw_ops;
40 }
41 EXPORT_SYMBOL(otx2_init_hw_ops);
42
cn10k_lmtst_init(struct otx2_nic * pfvf)43 int cn10k_lmtst_init(struct otx2_nic *pfvf)
44 {
45
46 struct lmtst_tbl_setup_req *req;
47 struct otx2_lmt_info *lmt_info;
48 int err, cpu;
49
50 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag))
51 return 0;
52
53 /* Total LMTLINES = num_online_cpus() * 32 (For Burst flush).*/
54 pfvf->tot_lmt_lines = (num_online_cpus() * LMT_BURST_SIZE);
55 pfvf->hw.lmt_info = alloc_percpu(struct otx2_lmt_info);
56
57 mutex_lock(&pfvf->mbox.lock);
58 req = otx2_mbox_alloc_msg_lmtst_tbl_setup(&pfvf->mbox);
59 if (!req) {
60 mutex_unlock(&pfvf->mbox.lock);
61 return -ENOMEM;
62 }
63
64 req->use_local_lmt_region = true;
65
66 err = qmem_alloc(pfvf->dev, &pfvf->dync_lmt, pfvf->tot_lmt_lines,
67 LMT_LINE_SIZE);
68 if (err) {
69 mutex_unlock(&pfvf->mbox.lock);
70 return err;
71 }
72 pfvf->hw.lmt_base = (u64 *)pfvf->dync_lmt->base;
73 req->lmt_iova = (u64)pfvf->dync_lmt->iova;
74
75 err = otx2_sync_mbox_msg(&pfvf->mbox);
76 mutex_unlock(&pfvf->mbox.lock);
77
78 for_each_possible_cpu(cpu) {
79 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, cpu);
80 lmt_info->lmt_addr = ((u64)pfvf->hw.lmt_base +
81 (cpu * LMT_BURST_SIZE * LMT_LINE_SIZE));
82 lmt_info->lmt_id = cpu * LMT_BURST_SIZE;
83 }
84
85 return 0;
86 }
87 EXPORT_SYMBOL(cn10k_lmtst_init);
88
cn10k_sq_aq_init(void * dev,u16 qidx,u8 chan_offset,u16 sqb_aura)89 int cn10k_sq_aq_init(void *dev, u16 qidx, u8 chan_offset, u16 sqb_aura)
90 {
91 struct nix_cn10k_aq_enq_req *aq;
92 struct otx2_nic *pfvf = dev;
93
94 /* Get memory to put this msg */
95 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
96 if (!aq)
97 return -ENOMEM;
98
99 aq->sq.cq = pfvf->hw.rx_queues + qidx;
100 aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */
101 aq->sq.cq_ena = 1;
102 aq->sq.ena = 1;
103 aq->sq.smq = otx2_get_smq_idx(pfvf, qidx);
104 aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen);
105 aq->sq.default_chan = pfvf->hw.tx_chan_base + chan_offset;
106 aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */
107 aq->sq.sqb_aura = sqb_aura;
108 aq->sq.sq_int_ena = NIX_SQINT_BITS;
109 aq->sq.qint_idx = 0;
110 /* Due pipelining impact minimum 2000 unused SQ CQE's
111 * need to maintain to avoid CQ overflow.
112 */
113 aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt));
114
115 /* Fill AQ info */
116 aq->qidx = qidx;
117 aq->ctype = NIX_AQ_CTYPE_SQ;
118 aq->op = NIX_AQ_INSTOP_INIT;
119
120 return otx2_sync_mbox_msg(&pfvf->mbox);
121 }
122
123 #define NPA_MAX_BURST 16
cn10k_refill_pool_ptrs(void * dev,struct otx2_cq_queue * cq)124 int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
125 {
126 struct otx2_nic *pfvf = dev;
127 int cnt = cq->pool_ptrs;
128 u64 ptrs[NPA_MAX_BURST];
129 struct otx2_pool *pool;
130 dma_addr_t bufptr;
131 int num_ptrs = 1;
132
133 pool = &pfvf->qset.pool[cq->cq_idx];
134
135 /* Refill pool with new buffers */
136 while (cq->pool_ptrs) {
137 if (otx2_alloc_buffer(pfvf, cq, &bufptr)) {
138 if (num_ptrs--)
139 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,
140 num_ptrs);
141 break;
142 }
143 cq->pool_ptrs--;
144 ptrs[num_ptrs] = pool->xsk_pool ?
145 (u64)bufptr : (u64)bufptr + OTX2_HEAD_ROOM;
146
147 num_ptrs++;
148 if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) {
149 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,
150 num_ptrs);
151 num_ptrs = 1;
152 }
153 }
154 return cnt - cq->pool_ptrs;
155 }
156
cn10k_sqe_flush(void * dev,struct otx2_snd_queue * sq,int size,int qidx)157 void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx)
158 {
159 struct otx2_lmt_info *lmt_info;
160 struct otx2_nic *pfvf = dev;
161 u64 val = 0, tar_addr = 0;
162
163 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id());
164 /* FIXME: val[0:10] LMT_ID.
165 * [12:15] no of LMTST - 1 in the burst.
166 * [19:63] data size of each LMTST in the burst except first.
167 */
168 val = (lmt_info->lmt_id & 0x7FF);
169 /* Target address for LMTST flush tells HW how many 128bit
170 * words are present.
171 * tar_addr[6:4] size of first LMTST - 1 in units of 128b.
172 */
173 tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4;
174 dma_wmb();
175 memcpy((u64 *)lmt_info->lmt_addr, sq->sqe_base, size);
176 cn10k_lmt_flush(val, tar_addr);
177
178 sq->head++;
179 sq->head &= (sq->sqe_cnt - 1);
180 }
181
cn10k_free_all_ipolicers(struct otx2_nic * pfvf)182 int cn10k_free_all_ipolicers(struct otx2_nic *pfvf)
183 {
184 struct nix_bandprof_free_req *req;
185 int rc;
186
187 if (is_dev_otx2(pfvf->pdev))
188 return 0;
189
190 mutex_lock(&pfvf->mbox.lock);
191
192 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox);
193 if (!req) {
194 rc = -ENOMEM;
195 goto out;
196 }
197
198 /* Free all bandwidth profiles allocated */
199 req->free_all = true;
200
201 rc = otx2_sync_mbox_msg(&pfvf->mbox);
202 out:
203 mutex_unlock(&pfvf->mbox.lock);
204 return rc;
205 }
206
cn10k_alloc_leaf_profile(struct otx2_nic * pfvf,u16 * leaf)207 int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf)
208 {
209 struct nix_bandprof_alloc_req *req;
210 struct nix_bandprof_alloc_rsp *rsp;
211 int rc;
212
213 req = otx2_mbox_alloc_msg_nix_bandprof_alloc(&pfvf->mbox);
214 if (!req)
215 return -ENOMEM;
216
217 req->prof_count[BAND_PROF_LEAF_LAYER] = 1;
218
219 rc = otx2_sync_mbox_msg(&pfvf->mbox);
220 if (rc)
221 goto out;
222
223 rsp = (struct nix_bandprof_alloc_rsp *)
224 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr);
225 if (IS_ERR(rsp)) {
226 rc = PTR_ERR(rsp);
227 goto out;
228 }
229
230 if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) {
231 rc = -EIO;
232 goto out;
233 }
234
235 *leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0];
236 out:
237 if (rc) {
238 dev_warn(pfvf->dev,
239 "Failed to allocate ingress bandwidth policer\n");
240 }
241
242 return rc;
243 }
244
cn10k_alloc_matchall_ipolicer(struct otx2_nic * pfvf)245 int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf)
246 {
247 struct otx2_hw *hw = &pfvf->hw;
248 int ret;
249
250 mutex_lock(&pfvf->mbox.lock);
251
252 ret = cn10k_alloc_leaf_profile(pfvf, &hw->matchall_ipolicer);
253
254 mutex_unlock(&pfvf->mbox.lock);
255
256 return ret;
257 }
258
259 #define POLICER_TIMESTAMP 1 /* 1 second */
260 #define MAX_RATE_EXP 22 /* Valid rate exponent range: 0 - 22 */
261
cn10k_get_ingress_burst_cfg(u32 burst,u32 * burst_exp,u32 * burst_mantissa)262 static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp,
263 u32 *burst_mantissa)
264 {
265 int tmp;
266
267 /* Burst is calculated as
268 * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT]
269 * This is the upper limit on number tokens (bytes) that
270 * can be accumulated in the bucket.
271 */
272 *burst_exp = ilog2(burst);
273 if (burst < 256) {
274 /* No float: can't express mantissa in this case */
275 *burst_mantissa = 0;
276 return;
277 }
278
279 if (*burst_exp > MAX_RATE_EXP)
280 *burst_exp = MAX_RATE_EXP;
281
282 /* Calculate mantissa
283 * Find remaining bytes 'burst - 2^burst_exp'
284 * mantissa = (remaining bytes) / 2^ (burst_exp - 8)
285 */
286 tmp = burst - rounddown_pow_of_two(burst);
287 *burst_mantissa = tmp / (1UL << (*burst_exp - 8));
288 }
289
cn10k_get_ingress_rate_cfg(u64 rate,u32 * rate_exp,u32 * rate_mantissa,u32 * rdiv)290 static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp,
291 u32 *rate_mantissa, u32 *rdiv)
292 {
293 u32 div = 0;
294 u32 exp = 0;
295 u64 tmp;
296
297 /* Figure out mantissa, exponent and divider from given max pkt rate
298 *
299 * To achieve desired rate HW adds
300 * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every
301 * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket.
302 * Here policer timeunit is 2 usecs and rate is in bits per sec.
303 * Since floating point cannot be used below algorithm uses 1000000
304 * scale factor to support rates upto 100Gbps.
305 */
306 tmp = rate * 32 * 2;
307 if (tmp < 256000000) {
308 while (tmp < 256000000) {
309 tmp = tmp * 2;
310 div++;
311 }
312 } else {
313 for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++)
314 tmp = tmp / 2;
315
316 if (exp > MAX_RATE_EXP)
317 exp = MAX_RATE_EXP;
318 }
319
320 *rate_mantissa = (tmp - 256000000) / 1000000;
321 *rate_exp = exp;
322 *rdiv = div;
323 }
324
cn10k_map_unmap_rq_policer(struct otx2_nic * pfvf,int rq_idx,u16 policer,bool map)325 int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx,
326 u16 policer, bool map)
327 {
328 struct nix_cn10k_aq_enq_req *aq;
329
330 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
331 if (!aq)
332 return -ENOMEM;
333
334 /* Enable policing and set the bandwidth profile (policer) index */
335 if (map)
336 aq->rq.policer_ena = 1;
337 else
338 aq->rq.policer_ena = 0;
339 aq->rq_mask.policer_ena = 1;
340
341 aq->rq.band_prof_id = policer;
342 aq->rq_mask.band_prof_id = GENMASK(9, 0);
343
344 /* If policer id is greater than 1023 then it implies hardware supports
345 * more leaf profiles. In that case use band_prof_id_h for 4 MSBs.
346 */
347 aq->rq.band_prof_id_h = policer >> 10;
348 aq->rq_mask.band_prof_id_h = GENMASK(3, 0);
349
350 /* Fill AQ info */
351 aq->qidx = rq_idx;
352 aq->ctype = NIX_AQ_CTYPE_RQ;
353 aq->op = NIX_AQ_INSTOP_WRITE;
354
355 return otx2_sync_mbox_msg(&pfvf->mbox);
356 }
357
cn10k_free_leaf_profile(struct otx2_nic * pfvf,u16 leaf)358 int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf)
359 {
360 struct nix_bandprof_free_req *req;
361
362 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox);
363 if (!req)
364 return -ENOMEM;
365
366 req->prof_count[BAND_PROF_LEAF_LAYER] = 1;
367 req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf;
368
369 return otx2_sync_mbox_msg(&pfvf->mbox);
370 }
371
cn10k_free_matchall_ipolicer(struct otx2_nic * pfvf)372 int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf)
373 {
374 struct otx2_hw *hw = &pfvf->hw;
375 int qidx, rc;
376
377 mutex_lock(&pfvf->mbox.lock);
378
379 /* Remove RQ's policer mapping */
380 for (qidx = 0; qidx < hw->rx_queues; qidx++) {
381 rc = cn10k_map_unmap_rq_policer(pfvf, qidx, hw->matchall_ipolicer, false);
382 if (rc)
383 dev_warn(pfvf->dev, "Failed to unmap RQ %d's policer (error %d).",
384 qidx, rc);
385 }
386
387 rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer);
388
389 mutex_unlock(&pfvf->mbox.lock);
390 return rc;
391 }
392
cn10k_set_ipolicer_rate(struct otx2_nic * pfvf,u16 profile,u32 burst,u64 rate,bool pps)393 int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile,
394 u32 burst, u64 rate, bool pps)
395 {
396 struct nix_cn10k_aq_enq_req *aq;
397 u32 burst_exp, burst_mantissa;
398 u32 rate_exp, rate_mantissa;
399 u32 rdiv;
400
401 /* Get exponent and mantissa values for the desired rate */
402 cn10k_get_ingress_burst_cfg(burst, &burst_exp, &burst_mantissa);
403 cn10k_get_ingress_rate_cfg(rate, &rate_exp, &rate_mantissa, &rdiv);
404
405 /* Init bandwidth profile */
406 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
407 if (!aq)
408 return -ENOMEM;
409
410 /* Set initial color mode to blind */
411 aq->prof.icolor = 0x03;
412 aq->prof_mask.icolor = 0x03;
413
414 /* Set rate and burst values */
415 aq->prof.cir_exponent = rate_exp;
416 aq->prof_mask.cir_exponent = 0x1F;
417
418 aq->prof.cir_mantissa = rate_mantissa;
419 aq->prof_mask.cir_mantissa = 0xFF;
420
421 aq->prof.cbs_exponent = burst_exp;
422 aq->prof_mask.cbs_exponent = 0x1F;
423
424 aq->prof.cbs_mantissa = burst_mantissa;
425 aq->prof_mask.cbs_mantissa = 0xFF;
426
427 aq->prof.rdiv = rdiv;
428 aq->prof_mask.rdiv = 0xF;
429
430 if (pps) {
431 /* The amount of decremented tokens is calculated according to
432 * the following equation:
433 * max([ LMODE ? 0 : (packet_length - LXPTR)] +
434 * ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT],
435 * 1/256)
436 * if LMODE is 1 then rate limiting will be based on
437 * PPS otherwise bps.
438 * The aim of the ADJUST value is to specify a token cost per
439 * packet in contrary to the packet length that specifies a
440 * cost per byte. To rate limit based on PPS adjust mantissa
441 * is set as 384 and exponent as 1 so that number of tokens
442 * decremented becomes 1 i.e, 1 token per packeet.
443 */
444 aq->prof.adjust_exponent = 1;
445 aq->prof_mask.adjust_exponent = 0x1F;
446
447 aq->prof.adjust_mantissa = 384;
448 aq->prof_mask.adjust_mantissa = 0x1FF;
449
450 aq->prof.lmode = 0x1;
451 aq->prof_mask.lmode = 0x1;
452 }
453
454 /* Two rate three color marker
455 * With PEIR/EIR set to zero, color will be either green or red
456 */
457 aq->prof.meter_algo = 2;
458 aq->prof_mask.meter_algo = 0x3;
459
460 aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP;
461 aq->prof_mask.rc_action = 0x3;
462
463 aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS;
464 aq->prof_mask.yc_action = 0x3;
465
466 aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS;
467 aq->prof_mask.gc_action = 0x3;
468
469 /* Setting exponent value as 24 and mantissa as 0 configures
470 * the bucket with zero values making bucket unused. Peak
471 * information rate and Excess information rate buckets are
472 * unused here.
473 */
474 aq->prof.peir_exponent = 24;
475 aq->prof_mask.peir_exponent = 0x1F;
476
477 aq->prof.peir_mantissa = 0;
478 aq->prof_mask.peir_mantissa = 0xFF;
479
480 aq->prof.pebs_exponent = 24;
481 aq->prof_mask.pebs_exponent = 0x1F;
482
483 aq->prof.pebs_mantissa = 0;
484 aq->prof_mask.pebs_mantissa = 0xFF;
485
486 aq->prof.hl_en = 0;
487 aq->prof_mask.hl_en = 1;
488
489 /* Fill AQ info */
490 aq->qidx = profile;
491 aq->ctype = NIX_AQ_CTYPE_BANDPROF;
492 aq->op = NIX_AQ_INSTOP_WRITE;
493
494 return otx2_sync_mbox_msg(&pfvf->mbox);
495 }
496
cn10k_set_matchall_ipolicer_rate(struct otx2_nic * pfvf,u32 burst,u64 rate)497 int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf,
498 u32 burst, u64 rate)
499 {
500 struct otx2_hw *hw = &pfvf->hw;
501 int qidx, rc;
502
503 mutex_lock(&pfvf->mbox.lock);
504
505 rc = cn10k_set_ipolicer_rate(pfvf, hw->matchall_ipolicer, burst,
506 rate, false);
507 if (rc)
508 goto out;
509
510 for (qidx = 0; qidx < hw->rx_queues; qidx++) {
511 rc = cn10k_map_unmap_rq_policer(pfvf, qidx,
512 hw->matchall_ipolicer, true);
513 if (rc)
514 break;
515 }
516
517 out:
518 mutex_unlock(&pfvf->mbox.lock);
519 return rc;
520 }
521