1 // SPDX-License-Identifier: GPL-2.0-only 2 // Copyright 2024 Cisco Systems, Inc. All rights reserved. 3 4 #include <linux/skbuff.h> 5 #include <linux/if_vlan.h> 6 #include <net/busy_poll.h> 7 #include "enic.h" 8 #include "enic_res.h" 9 #include "enic_rq.h" 10 #include "vnic_rq.h" 11 #include "cq_enet_desc.h" 12 13 #define ENIC_LARGE_PKT_THRESHOLD 1000 14 15 static void enic_intr_update_pkt_size(struct vnic_rx_bytes_counter *pkt_size, 16 u32 pkt_len) 17 { 18 if (pkt_len > ENIC_LARGE_PKT_THRESHOLD) 19 pkt_size->large_pkt_bytes_cnt += pkt_len; 20 else 21 pkt_size->small_pkt_bytes_cnt += pkt_len; 22 } 23 24 static void enic_rq_cq_desc_dec(void *cq_desc, u8 cq_desc_size, u8 *type, 25 u8 *color, u16 *q_number, u16 *completed_index) 26 { 27 /* type_color is the last field for all cq structs */ 28 u8 type_color; 29 30 switch (cq_desc_size) { 31 case VNIC_RQ_CQ_ENTRY_SIZE_16: { 32 struct cq_enet_rq_desc *desc = 33 (struct cq_enet_rq_desc *)cq_desc; 34 type_color = desc->type_color; 35 36 /* Make sure color bit is read from desc *before* other fields 37 * are read from desc. Hardware guarantees color bit is last 38 * bit (byte) written. Adding the rmb() prevents the compiler 39 * and/or CPU from reordering the reads which would potentially 40 * result in reading stale values. 41 */ 42 rmb(); 43 44 *q_number = le16_to_cpu(desc->q_number_rss_type_flags) & 45 CQ_DESC_Q_NUM_MASK; 46 *completed_index = le16_to_cpu(desc->completed_index_flags) & 47 CQ_DESC_COMP_NDX_MASK; 48 break; 49 } 50 case VNIC_RQ_CQ_ENTRY_SIZE_32: { 51 struct cq_enet_rq_desc_32 *desc = 52 (struct cq_enet_rq_desc_32 *)cq_desc; 53 type_color = desc->type_color; 54 55 /* Make sure color bit is read from desc *before* other fields 56 * are read from desc. Hardware guarantees color bit is last 57 * bit (byte) written. Adding the rmb() prevents the compiler 58 * and/or CPU from reordering the reads which would potentially 59 * result in reading stale values. 60 */ 61 rmb(); 62 63 *q_number = le16_to_cpu(desc->q_number_rss_type_flags) & 64 CQ_DESC_Q_NUM_MASK; 65 *completed_index = le16_to_cpu(desc->completed_index_flags) & 66 CQ_DESC_COMP_NDX_MASK; 67 *completed_index |= (desc->fetch_index_flags & CQ_DESC_32_FI_MASK) << 68 CQ_DESC_COMP_NDX_BITS; 69 break; 70 } 71 case VNIC_RQ_CQ_ENTRY_SIZE_64: { 72 struct cq_enet_rq_desc_64 *desc = 73 (struct cq_enet_rq_desc_64 *)cq_desc; 74 type_color = desc->type_color; 75 76 /* Make sure color bit is read from desc *before* other fields 77 * are read from desc. Hardware guarantees color bit is last 78 * bit (byte) written. Adding the rmb() prevents the compiler 79 * and/or CPU from reordering the reads which would potentially 80 * result in reading stale values. 81 */ 82 rmb(); 83 84 *q_number = le16_to_cpu(desc->q_number_rss_type_flags) & 85 CQ_DESC_Q_NUM_MASK; 86 *completed_index = le16_to_cpu(desc->completed_index_flags) & 87 CQ_DESC_COMP_NDX_MASK; 88 *completed_index |= (desc->fetch_index_flags & CQ_DESC_64_FI_MASK) << 89 CQ_DESC_COMP_NDX_BITS; 90 break; 91 } 92 } 93 94 *color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK; 95 *type = type_color & CQ_DESC_TYPE_MASK; 96 } 97 98 static void enic_rq_set_skb_flags(struct vnic_rq *vrq, u8 type, u32 rss_hash, 99 u8 rss_type, u8 fcoe, u8 fcoe_fc_crc_ok, 100 u8 vlan_stripped, u8 csum_not_calc, 101 u8 tcp_udp_csum_ok, u8 ipv6, u8 ipv4_csum_ok, 102 u16 vlan_tci, struct sk_buff *skb) 103 { 104 struct enic *enic = vnic_dev_priv(vrq->vdev); 105 struct net_device *netdev = enic->netdev; 106 struct enic_rq_stats *rqstats = &enic->rq[vrq->index].stats; 107 bool outer_csum_ok = true, encap = false; 108 109 if ((netdev->features & NETIF_F_RXHASH) && rss_hash && type == 3) { 110 switch (rss_type) { 111 case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv4: 112 case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6: 113 case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6_EX: 114 skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L4); 115 rqstats->l4_rss_hash++; 116 break; 117 case CQ_ENET_RQ_DESC_RSS_TYPE_IPv4: 118 case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6: 119 case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6_EX: 120 skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L3); 121 rqstats->l3_rss_hash++; 122 break; 123 } 124 } 125 if (enic->vxlan.vxlan_udp_port_number) { 126 switch (enic->vxlan.patch_level) { 127 case 0: 128 if (fcoe) { 129 encap = true; 130 outer_csum_ok = fcoe_fc_crc_ok; 131 } 132 break; 133 case 2: 134 if (type == 7 && (rss_hash & BIT(0))) { 135 encap = true; 136 outer_csum_ok = (rss_hash & BIT(1)) && 137 (rss_hash & BIT(2)); 138 } 139 break; 140 } 141 } 142 143 /* Hardware does not provide whole packet checksum. It only 144 * provides pseudo checksum. Since hw validates the packet 145 * checksum but not provide us the checksum value. use 146 * CHECSUM_UNNECESSARY. 147 * 148 * In case of encap pkt tcp_udp_csum_ok/tcp_udp_csum_ok is 149 * inner csum_ok. outer_csum_ok is set by hw when outer udp 150 * csum is correct or is zero. 151 */ 152 if ((netdev->features & NETIF_F_RXCSUM) && !csum_not_calc && 153 tcp_udp_csum_ok && outer_csum_ok && (ipv4_csum_ok || ipv6)) { 154 skb->ip_summed = CHECKSUM_UNNECESSARY; 155 skb->csum_level = encap; 156 if (encap) 157 rqstats->csum_unnecessary_encap++; 158 else 159 rqstats->csum_unnecessary++; 160 } 161 162 if (vlan_stripped) { 163 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci); 164 rqstats->vlan_stripped++; 165 } 166 } 167 168 /* 169 * cq_enet_rq_desc accesses section uses only the 1st 15 bytes of the cq which 170 * is identical for all type (16,32 and 64 byte) of cqs. 171 */ 172 static void cq_enet_rq_desc_dec(struct cq_enet_rq_desc *desc, u8 *ingress_port, 173 u8 *fcoe, u8 *eop, u8 *sop, u8 *rss_type, 174 u8 *csum_not_calc, u32 *rss_hash, 175 u16 *bytes_written, u8 *packet_error, 176 u8 *vlan_stripped, u16 *vlan_tci, 177 u16 *checksum, u8 *fcoe_sof, 178 u8 *fcoe_fc_crc_ok, u8 *fcoe_enc_error, 179 u8 *fcoe_eof, u8 *tcp_udp_csum_ok, u8 *udp, 180 u8 *tcp, u8 *ipv4_csum_ok, u8 *ipv6, u8 *ipv4, 181 u8 *ipv4_fragment, u8 *fcs_ok) 182 { 183 u16 completed_index_flags; 184 u16 q_number_rss_type_flags; 185 u16 bytes_written_flags; 186 187 completed_index_flags = le16_to_cpu(desc->completed_index_flags); 188 q_number_rss_type_flags = 189 le16_to_cpu(desc->q_number_rss_type_flags); 190 bytes_written_flags = le16_to_cpu(desc->bytes_written_flags); 191 192 *ingress_port = (completed_index_flags & 193 CQ_ENET_RQ_DESC_FLAGS_INGRESS_PORT) ? 1 : 0; 194 *fcoe = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_FCOE) ? 195 1 : 0; 196 *eop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_EOP) ? 197 1 : 0; 198 *sop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_SOP) ? 199 1 : 0; 200 201 *rss_type = (u8)((q_number_rss_type_flags >> CQ_DESC_Q_NUM_BITS) & 202 CQ_ENET_RQ_DESC_RSS_TYPE_MASK); 203 *csum_not_calc = (q_number_rss_type_flags & 204 CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ? 1 : 0; 205 206 *rss_hash = le32_to_cpu(desc->rss_hash); 207 208 *bytes_written = bytes_written_flags & 209 CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; 210 *packet_error = (bytes_written_flags & 211 CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ? 1 : 0; 212 *vlan_stripped = (bytes_written_flags & 213 CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) ? 1 : 0; 214 215 /* 216 * Tag Control Information(16) = user_priority(3) + cfi(1) + vlan(12) 217 */ 218 *vlan_tci = le16_to_cpu(desc->vlan); 219 220 if (*fcoe) { 221 *fcoe_sof = (u8)(le16_to_cpu(desc->checksum_fcoe) & 222 CQ_ENET_RQ_DESC_FCOE_SOF_MASK); 223 *fcoe_fc_crc_ok = (desc->flags & 224 CQ_ENET_RQ_DESC_FCOE_FC_CRC_OK) ? 1 : 0; 225 *fcoe_enc_error = (desc->flags & 226 CQ_ENET_RQ_DESC_FCOE_ENC_ERROR) ? 1 : 0; 227 *fcoe_eof = (u8)((le16_to_cpu(desc->checksum_fcoe) >> 228 CQ_ENET_RQ_DESC_FCOE_EOF_SHIFT) & 229 CQ_ENET_RQ_DESC_FCOE_EOF_MASK); 230 *checksum = 0; 231 } else { 232 *fcoe_sof = 0; 233 *fcoe_fc_crc_ok = 0; 234 *fcoe_enc_error = 0; 235 *fcoe_eof = 0; 236 *checksum = le16_to_cpu(desc->checksum_fcoe); 237 } 238 239 *tcp_udp_csum_ok = 240 (desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ? 1 : 0; 241 *udp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_UDP) ? 1 : 0; 242 *tcp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP) ? 1 : 0; 243 *ipv4_csum_ok = 244 (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ? 1 : 0; 245 *ipv6 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV6) ? 1 : 0; 246 *ipv4 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4) ? 1 : 0; 247 *ipv4_fragment = 248 (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT) ? 1 : 0; 249 *fcs_ok = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) ? 1 : 0; 250 } 251 252 static bool enic_rq_pkt_error(struct vnic_rq *vrq, u8 packet_error, u8 fcs_ok, 253 u16 bytes_written) 254 { 255 struct enic *enic = vnic_dev_priv(vrq->vdev); 256 struct enic_rq_stats *rqstats = &enic->rq[vrq->index].stats; 257 258 if (packet_error) { 259 if (!fcs_ok) { 260 if (bytes_written > 0) 261 rqstats->bad_fcs++; 262 else if (bytes_written == 0) 263 rqstats->pkt_truncated++; 264 } 265 return true; 266 } 267 return false; 268 } 269 270 int enic_rq_alloc_buf(struct vnic_rq *rq) 271 { 272 struct enic *enic = vnic_dev_priv(rq->vdev); 273 struct net_device *netdev = enic->netdev; 274 struct enic_rq *erq = &enic->rq[rq->index]; 275 struct enic_rq_stats *rqstats = &erq->stats; 276 unsigned int offset = 0; 277 unsigned int len = netdev->mtu + VLAN_ETH_HLEN; 278 unsigned int os_buf_index = 0; 279 dma_addr_t dma_addr; 280 struct vnic_rq_buf *buf = rq->to_use; 281 struct page *page; 282 unsigned int truesize = len; 283 284 if (buf->os_buf) { 285 enic_queue_rq_desc(rq, buf->os_buf, os_buf_index, buf->dma_addr, 286 buf->len); 287 288 return 0; 289 } 290 291 page = page_pool_dev_alloc(erq->pool, &offset, &truesize); 292 if (unlikely(!page)) { 293 rqstats->pp_alloc_fail++; 294 return -ENOMEM; 295 } 296 buf->offset = offset; 297 buf->truesize = truesize; 298 dma_addr = page_pool_get_dma_addr(page) + offset; 299 enic_queue_rq_desc(rq, (void *)page, os_buf_index, dma_addr, len); 300 301 return 0; 302 } 303 304 void enic_free_rq_buf(struct vnic_rq *rq, struct vnic_rq_buf *buf) 305 { 306 struct enic *enic = vnic_dev_priv(rq->vdev); 307 struct enic_rq *erq = &enic->rq[rq->index]; 308 309 if (!buf->os_buf) 310 return; 311 312 page_pool_put_full_page(erq->pool, (struct page *)buf->os_buf, true); 313 buf->os_buf = NULL; 314 } 315 316 static void enic_rq_indicate_buf(struct enic *enic, struct vnic_rq *rq, 317 struct vnic_rq_buf *buf, void *cq_desc, 318 u8 type, u16 q_number, u16 completed_index) 319 { 320 struct sk_buff *skb; 321 struct vnic_cq *cq = &enic->cq[enic_cq_rq(enic, rq->index)]; 322 struct enic_rq_stats *rqstats = &enic->rq[rq->index].stats; 323 struct napi_struct *napi; 324 325 u8 eop, sop, ingress_port, vlan_stripped; 326 u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof; 327 u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok; 328 u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc; 329 u8 packet_error; 330 u16 bytes_written, vlan_tci, checksum; 331 u32 rss_hash; 332 333 rqstats->packets++; 334 335 cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, &ingress_port, 336 &fcoe, &eop, &sop, &rss_type, &csum_not_calc, 337 &rss_hash, &bytes_written, &packet_error, 338 &vlan_stripped, &vlan_tci, &checksum, &fcoe_sof, 339 &fcoe_fc_crc_ok, &fcoe_enc_error, &fcoe_eof, 340 &tcp_udp_csum_ok, &udp, &tcp, &ipv4_csum_ok, &ipv6, 341 &ipv4, &ipv4_fragment, &fcs_ok); 342 343 if (enic_rq_pkt_error(rq, packet_error, fcs_ok, bytes_written)) 344 return; 345 346 if (eop && bytes_written > 0) { 347 /* Good receive 348 */ 349 rqstats->bytes += bytes_written; 350 napi = &enic->napi[rq->index]; 351 skb = napi_get_frags(napi); 352 if (unlikely(!skb)) { 353 net_warn_ratelimited("%s: skb alloc error rq[%d], desc[%d]\n", 354 enic->netdev->name, rq->index, 355 completed_index); 356 rqstats->no_skb++; 357 return; 358 } 359 360 prefetch(skb->data - NET_IP_ALIGN); 361 362 dma_sync_single_for_cpu(&enic->pdev->dev, buf->dma_addr, 363 bytes_written, DMA_FROM_DEVICE); 364 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 365 (struct page *)buf->os_buf, buf->offset, 366 bytes_written, buf->truesize); 367 skb_record_rx_queue(skb, q_number); 368 enic_rq_set_skb_flags(rq, type, rss_hash, rss_type, fcoe, 369 fcoe_fc_crc_ok, vlan_stripped, 370 csum_not_calc, tcp_udp_csum_ok, ipv6, 371 ipv4_csum_ok, vlan_tci, skb); 372 skb_mark_for_recycle(skb); 373 napi_gro_frags(napi); 374 if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce) 375 enic_intr_update_pkt_size(&cq->pkt_size_counter, 376 bytes_written); 377 buf->os_buf = NULL; 378 buf->dma_addr = 0; 379 buf = buf->next; 380 } else { 381 /* Buffer overflow 382 */ 383 rqstats->pkt_truncated++; 384 } 385 } 386 387 static void enic_rq_service(struct enic *enic, void *cq_desc, u8 type, 388 u16 q_number, u16 completed_index) 389 { 390 struct enic_rq_stats *rqstats = &enic->rq[q_number].stats; 391 struct vnic_rq *vrq = &enic->rq[q_number].vrq; 392 struct vnic_rq_buf *vrq_buf = vrq->to_clean; 393 int skipped; 394 395 while (1) { 396 skipped = (vrq_buf->index != completed_index); 397 if (!skipped) 398 enic_rq_indicate_buf(enic, vrq, vrq_buf, cq_desc, type, 399 q_number, completed_index); 400 else 401 rqstats->desc_skip++; 402 403 vrq->ring.desc_avail++; 404 vrq->to_clean = vrq_buf->next; 405 vrq_buf = vrq_buf->next; 406 if (!skipped) 407 break; 408 } 409 } 410 411 unsigned int enic_rq_cq_service(struct enic *enic, unsigned int cq_index, 412 unsigned int work_to_do) 413 { 414 struct vnic_cq *cq = &enic->cq[cq_index]; 415 void *cq_desc = vnic_cq_to_clean(cq); 416 u16 q_number, completed_index; 417 unsigned int work_done = 0; 418 u8 type, color; 419 420 enic_rq_cq_desc_dec(cq_desc, enic->ext_cq, &type, &color, &q_number, 421 &completed_index); 422 423 while (color != cq->last_color) { 424 enic_rq_service(enic, cq_desc, type, q_number, completed_index); 425 vnic_cq_inc_to_clean(cq); 426 427 if (++work_done >= work_to_do) 428 break; 429 430 cq_desc = vnic_cq_to_clean(cq); 431 enic_rq_cq_desc_dec(cq_desc, enic->ext_cq, &type, &color, 432 &q_number, &completed_index); 433 } 434 435 return work_done; 436 } 437