1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */
3
4 #include <linux/etherdevice.h>
5 #include <net/ip6_checksum.h>
6 #include <net/page_pool/helpers.h>
7 #include <net/inet_ecn.h>
8 #include <linux/iopoll.h>
9 #include <linux/sctp.h>
10 #include <linux/pci.h>
11 #include <net/tcp.h>
12 #include <net/ip.h>
13
14 #include "wx_type.h"
15 #include "wx_lib.h"
16 #include "wx_ptp.h"
17 #include "wx_hw.h"
18
19 /* Lookup table mapping the HW PTYPE to the bit field for decoding */
20 static struct wx_dec_ptype wx_ptype_lookup[256] = {
21 /* L2: mac */
22 [0x11] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
23 [0x12] = WX_PTT(L2, NONE, NONE, NONE, TS, PAY2),
24 [0x13] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
25 [0x14] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
26 [0x15] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
27 [0x16] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
28 [0x17] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
29
30 /* L2: ethertype filter */
31 [0x18 ... 0x1F] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
32
33 /* L3: ip non-tunnel */
34 [0x21] = WX_PTT(IP, FGV4, NONE, NONE, NONE, PAY3),
35 [0x22] = WX_PTT(IP, IPV4, NONE, NONE, NONE, PAY3),
36 [0x23] = WX_PTT(IP, IPV4, NONE, NONE, UDP, PAY4),
37 [0x24] = WX_PTT(IP, IPV4, NONE, NONE, TCP, PAY4),
38 [0x25] = WX_PTT(IP, IPV4, NONE, NONE, SCTP, PAY4),
39 [0x29] = WX_PTT(IP, FGV6, NONE, NONE, NONE, PAY3),
40 [0x2A] = WX_PTT(IP, IPV6, NONE, NONE, NONE, PAY3),
41 [0x2B] = WX_PTT(IP, IPV6, NONE, NONE, UDP, PAY3),
42 [0x2C] = WX_PTT(IP, IPV6, NONE, NONE, TCP, PAY4),
43 [0x2D] = WX_PTT(IP, IPV6, NONE, NONE, SCTP, PAY4),
44
45 /* L2: fcoe */
46 [0x30 ... 0x34] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
47 [0x38 ... 0x3C] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
48
49 /* IPv4 --> IPv4/IPv6 */
50 [0x81] = WX_PTT(IP, IPV4, IPIP, FGV4, NONE, PAY3),
51 [0x82] = WX_PTT(IP, IPV4, IPIP, IPV4, NONE, PAY3),
52 [0x83] = WX_PTT(IP, IPV4, IPIP, IPV4, UDP, PAY4),
53 [0x84] = WX_PTT(IP, IPV4, IPIP, IPV4, TCP, PAY4),
54 [0x85] = WX_PTT(IP, IPV4, IPIP, IPV4, SCTP, PAY4),
55 [0x89] = WX_PTT(IP, IPV4, IPIP, FGV6, NONE, PAY3),
56 [0x8A] = WX_PTT(IP, IPV4, IPIP, IPV6, NONE, PAY3),
57 [0x8B] = WX_PTT(IP, IPV4, IPIP, IPV6, UDP, PAY4),
58 [0x8C] = WX_PTT(IP, IPV4, IPIP, IPV6, TCP, PAY4),
59 [0x8D] = WX_PTT(IP, IPV4, IPIP, IPV6, SCTP, PAY4),
60
61 /* IPv4 --> GRE/NAT --> NONE/IPv4/IPv6 */
62 [0x90] = WX_PTT(IP, IPV4, IG, NONE, NONE, PAY3),
63 [0x91] = WX_PTT(IP, IPV4, IG, FGV4, NONE, PAY3),
64 [0x92] = WX_PTT(IP, IPV4, IG, IPV4, NONE, PAY3),
65 [0x93] = WX_PTT(IP, IPV4, IG, IPV4, UDP, PAY4),
66 [0x94] = WX_PTT(IP, IPV4, IG, IPV4, TCP, PAY4),
67 [0x95] = WX_PTT(IP, IPV4, IG, IPV4, SCTP, PAY4),
68 [0x99] = WX_PTT(IP, IPV4, IG, FGV6, NONE, PAY3),
69 [0x9A] = WX_PTT(IP, IPV4, IG, IPV6, NONE, PAY3),
70 [0x9B] = WX_PTT(IP, IPV4, IG, IPV6, UDP, PAY4),
71 [0x9C] = WX_PTT(IP, IPV4, IG, IPV6, TCP, PAY4),
72 [0x9D] = WX_PTT(IP, IPV4, IG, IPV6, SCTP, PAY4),
73
74 /* IPv4 --> GRE/NAT --> MAC --> NONE/IPv4/IPv6 */
75 [0xA0] = WX_PTT(IP, IPV4, IGM, NONE, NONE, PAY3),
76 [0xA1] = WX_PTT(IP, IPV4, IGM, FGV4, NONE, PAY3),
77 [0xA2] = WX_PTT(IP, IPV4, IGM, IPV4, NONE, PAY3),
78 [0xA3] = WX_PTT(IP, IPV4, IGM, IPV4, UDP, PAY4),
79 [0xA4] = WX_PTT(IP, IPV4, IGM, IPV4, TCP, PAY4),
80 [0xA5] = WX_PTT(IP, IPV4, IGM, IPV4, SCTP, PAY4),
81 [0xA9] = WX_PTT(IP, IPV4, IGM, FGV6, NONE, PAY3),
82 [0xAA] = WX_PTT(IP, IPV4, IGM, IPV6, NONE, PAY3),
83 [0xAB] = WX_PTT(IP, IPV4, IGM, IPV6, UDP, PAY4),
84 [0xAC] = WX_PTT(IP, IPV4, IGM, IPV6, TCP, PAY4),
85 [0xAD] = WX_PTT(IP, IPV4, IGM, IPV6, SCTP, PAY4),
86
87 /* IPv4 --> GRE/NAT --> MAC+VLAN --> NONE/IPv4/IPv6 */
88 [0xB0] = WX_PTT(IP, IPV4, IGMV, NONE, NONE, PAY3),
89 [0xB1] = WX_PTT(IP, IPV4, IGMV, FGV4, NONE, PAY3),
90 [0xB2] = WX_PTT(IP, IPV4, IGMV, IPV4, NONE, PAY3),
91 [0xB3] = WX_PTT(IP, IPV4, IGMV, IPV4, UDP, PAY4),
92 [0xB4] = WX_PTT(IP, IPV4, IGMV, IPV4, TCP, PAY4),
93 [0xB5] = WX_PTT(IP, IPV4, IGMV, IPV4, SCTP, PAY4),
94 [0xB9] = WX_PTT(IP, IPV4, IGMV, FGV6, NONE, PAY3),
95 [0xBA] = WX_PTT(IP, IPV4, IGMV, IPV6, NONE, PAY3),
96 [0xBB] = WX_PTT(IP, IPV4, IGMV, IPV6, UDP, PAY4),
97 [0xBC] = WX_PTT(IP, IPV4, IGMV, IPV6, TCP, PAY4),
98 [0xBD] = WX_PTT(IP, IPV4, IGMV, IPV6, SCTP, PAY4),
99
100 /* IPv6 --> IPv4/IPv6 */
101 [0xC1] = WX_PTT(IP, IPV6, IPIP, FGV4, NONE, PAY3),
102 [0xC2] = WX_PTT(IP, IPV6, IPIP, IPV4, NONE, PAY3),
103 [0xC3] = WX_PTT(IP, IPV6, IPIP, IPV4, UDP, PAY4),
104 [0xC4] = WX_PTT(IP, IPV6, IPIP, IPV4, TCP, PAY4),
105 [0xC5] = WX_PTT(IP, IPV6, IPIP, IPV4, SCTP, PAY4),
106 [0xC9] = WX_PTT(IP, IPV6, IPIP, FGV6, NONE, PAY3),
107 [0xCA] = WX_PTT(IP, IPV6, IPIP, IPV6, NONE, PAY3),
108 [0xCB] = WX_PTT(IP, IPV6, IPIP, IPV6, UDP, PAY4),
109 [0xCC] = WX_PTT(IP, IPV6, IPIP, IPV6, TCP, PAY4),
110 [0xCD] = WX_PTT(IP, IPV6, IPIP, IPV6, SCTP, PAY4),
111
112 /* IPv6 --> GRE/NAT -> NONE/IPv4/IPv6 */
113 [0xD0] = WX_PTT(IP, IPV6, IG, NONE, NONE, PAY3),
114 [0xD1] = WX_PTT(IP, IPV6, IG, FGV4, NONE, PAY3),
115 [0xD2] = WX_PTT(IP, IPV6, IG, IPV4, NONE, PAY3),
116 [0xD3] = WX_PTT(IP, IPV6, IG, IPV4, UDP, PAY4),
117 [0xD4] = WX_PTT(IP, IPV6, IG, IPV4, TCP, PAY4),
118 [0xD5] = WX_PTT(IP, IPV6, IG, IPV4, SCTP, PAY4),
119 [0xD9] = WX_PTT(IP, IPV6, IG, FGV6, NONE, PAY3),
120 [0xDA] = WX_PTT(IP, IPV6, IG, IPV6, NONE, PAY3),
121 [0xDB] = WX_PTT(IP, IPV6, IG, IPV6, UDP, PAY4),
122 [0xDC] = WX_PTT(IP, IPV6, IG, IPV6, TCP, PAY4),
123 [0xDD] = WX_PTT(IP, IPV6, IG, IPV6, SCTP, PAY4),
124
125 /* IPv6 --> GRE/NAT -> MAC -> NONE/IPv4/IPv6 */
126 [0xE0] = WX_PTT(IP, IPV6, IGM, NONE, NONE, PAY3),
127 [0xE1] = WX_PTT(IP, IPV6, IGM, FGV4, NONE, PAY3),
128 [0xE2] = WX_PTT(IP, IPV6, IGM, IPV4, NONE, PAY3),
129 [0xE3] = WX_PTT(IP, IPV6, IGM, IPV4, UDP, PAY4),
130 [0xE4] = WX_PTT(IP, IPV6, IGM, IPV4, TCP, PAY4),
131 [0xE5] = WX_PTT(IP, IPV6, IGM, IPV4, SCTP, PAY4),
132 [0xE9] = WX_PTT(IP, IPV6, IGM, FGV6, NONE, PAY3),
133 [0xEA] = WX_PTT(IP, IPV6, IGM, IPV6, NONE, PAY3),
134 [0xEB] = WX_PTT(IP, IPV6, IGM, IPV6, UDP, PAY4),
135 [0xEC] = WX_PTT(IP, IPV6, IGM, IPV6, TCP, PAY4),
136 [0xED] = WX_PTT(IP, IPV6, IGM, IPV6, SCTP, PAY4),
137
138 /* IPv6 --> GRE/NAT -> MAC--> NONE/IPv */
139 [0xF0] = WX_PTT(IP, IPV6, IGMV, NONE, NONE, PAY3),
140 [0xF1] = WX_PTT(IP, IPV6, IGMV, FGV4, NONE, PAY3),
141 [0xF2] = WX_PTT(IP, IPV6, IGMV, IPV4, NONE, PAY3),
142 [0xF3] = WX_PTT(IP, IPV6, IGMV, IPV4, UDP, PAY4),
143 [0xF4] = WX_PTT(IP, IPV6, IGMV, IPV4, TCP, PAY4),
144 [0xF5] = WX_PTT(IP, IPV6, IGMV, IPV4, SCTP, PAY4),
145 [0xF9] = WX_PTT(IP, IPV6, IGMV, FGV6, NONE, PAY3),
146 [0xFA] = WX_PTT(IP, IPV6, IGMV, IPV6, NONE, PAY3),
147 [0xFB] = WX_PTT(IP, IPV6, IGMV, IPV6, UDP, PAY4),
148 [0xFC] = WX_PTT(IP, IPV6, IGMV, IPV6, TCP, PAY4),
149 [0xFD] = WX_PTT(IP, IPV6, IGMV, IPV6, SCTP, PAY4),
150 };
151
wx_decode_ptype(const u8 ptype)152 struct wx_dec_ptype wx_decode_ptype(const u8 ptype)
153 {
154 return wx_ptype_lookup[ptype];
155 }
156 EXPORT_SYMBOL(wx_decode_ptype);
157
158 /* wx_test_staterr - tests bits in Rx descriptor status and error fields */
wx_test_staterr(union wx_rx_desc * rx_desc,const u32 stat_err_bits)159 static __le32 wx_test_staterr(union wx_rx_desc *rx_desc,
160 const u32 stat_err_bits)
161 {
162 return rx_desc->wb.upper.status_error & cpu_to_le32(stat_err_bits);
163 }
164
wx_dma_sync_frag(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer)165 static void wx_dma_sync_frag(struct wx_ring *rx_ring,
166 struct wx_rx_buffer *rx_buffer)
167 {
168 struct sk_buff *skb = rx_buffer->skb;
169 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
170
171 dma_sync_single_range_for_cpu(rx_ring->dev,
172 WX_CB(skb)->dma,
173 skb_frag_off(frag),
174 skb_frag_size(frag),
175 DMA_FROM_DEVICE);
176
177 /* If the page was released, just unmap it. */
178 if (unlikely(WX_CB(skb)->page_released))
179 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
180 }
181
wx_get_rx_buffer(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff ** skb,int * rx_buffer_pgcnt)182 static struct wx_rx_buffer *wx_get_rx_buffer(struct wx_ring *rx_ring,
183 union wx_rx_desc *rx_desc,
184 struct sk_buff **skb,
185 int *rx_buffer_pgcnt)
186 {
187 struct wx_rx_buffer *rx_buffer;
188 unsigned int size;
189
190 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
191 size = le16_to_cpu(rx_desc->wb.upper.length);
192
193 #if (PAGE_SIZE < 8192)
194 *rx_buffer_pgcnt = page_count(rx_buffer->page);
195 #else
196 *rx_buffer_pgcnt = 0;
197 #endif
198
199 prefetchw(rx_buffer->page);
200 *skb = rx_buffer->skb;
201
202 /* Delay unmapping of the first packet. It carries the header
203 * information, HW may still access the header after the writeback.
204 * Only unmap it when EOP is reached
205 */
206 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)) {
207 if (!*skb)
208 goto skip_sync;
209 } else {
210 if (*skb)
211 wx_dma_sync_frag(rx_ring, rx_buffer);
212 }
213
214 /* we are reusing so sync this buffer for CPU use */
215 dma_sync_single_range_for_cpu(rx_ring->dev,
216 rx_buffer->dma,
217 rx_buffer->page_offset,
218 size,
219 DMA_FROM_DEVICE);
220 skip_sync:
221 return rx_buffer;
222 }
223
wx_put_rx_buffer(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer,struct sk_buff * skb,int rx_buffer_pgcnt)224 static void wx_put_rx_buffer(struct wx_ring *rx_ring,
225 struct wx_rx_buffer *rx_buffer,
226 struct sk_buff *skb,
227 int rx_buffer_pgcnt)
228 {
229 if (!IS_ERR(skb) && WX_CB(skb)->dma == rx_buffer->dma)
230 /* the page has been released from the ring */
231 WX_CB(skb)->page_released = true;
232
233 /* clear contents of rx_buffer */
234 rx_buffer->page = NULL;
235 rx_buffer->skb = NULL;
236 }
237
wx_build_skb(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer,union wx_rx_desc * rx_desc)238 static struct sk_buff *wx_build_skb(struct wx_ring *rx_ring,
239 struct wx_rx_buffer *rx_buffer,
240 union wx_rx_desc *rx_desc)
241 {
242 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
243 #if (PAGE_SIZE < 8192)
244 unsigned int truesize = WX_RX_BUFSZ;
245 #else
246 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
247 #endif
248 struct sk_buff *skb = rx_buffer->skb;
249
250 if (!skb) {
251 void *page_addr = page_address(rx_buffer->page) +
252 rx_buffer->page_offset;
253
254 /* prefetch first cache line of first page */
255 net_prefetch(page_addr);
256
257 /* allocate a skb to store the frags */
258 skb = napi_alloc_skb(&rx_ring->q_vector->napi, WX_RXBUFFER_256);
259 if (unlikely(!skb))
260 return NULL;
261
262 /* we will be copying header into skb->data in
263 * pskb_may_pull so it is in our interest to prefetch
264 * it now to avoid a possible cache miss
265 */
266 prefetchw(skb->data);
267
268 if (size <= WX_RXBUFFER_256) {
269 memcpy(__skb_put(skb, size), page_addr,
270 ALIGN(size, sizeof(long)));
271 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, true);
272 return skb;
273 }
274
275 skb_mark_for_recycle(skb);
276
277 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP))
278 WX_CB(skb)->dma = rx_buffer->dma;
279
280 skb_add_rx_frag(skb, 0, rx_buffer->page,
281 rx_buffer->page_offset,
282 size, truesize);
283 goto out;
284
285 } else {
286 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
287 rx_buffer->page_offset, size, truesize);
288 }
289
290 out:
291 #if (PAGE_SIZE < 8192)
292 /* flip page offset to other buffer */
293 rx_buffer->page_offset ^= truesize;
294 #else
295 /* move offset up to the next cache line */
296 rx_buffer->page_offset += truesize;
297 #endif
298
299 return skb;
300 }
301
wx_alloc_mapped_page(struct wx_ring * rx_ring,struct wx_rx_buffer * bi)302 static bool wx_alloc_mapped_page(struct wx_ring *rx_ring,
303 struct wx_rx_buffer *bi)
304 {
305 struct page *page = bi->page;
306 dma_addr_t dma;
307
308 /* since we are recycling buffers we should seldom need to alloc */
309 if (likely(page))
310 return true;
311
312 page = page_pool_dev_alloc_pages(rx_ring->page_pool);
313 if (unlikely(!page))
314 return false;
315 dma = page_pool_get_dma_addr(page);
316
317 bi->page_dma = dma;
318 bi->page = page;
319 bi->page_offset = 0;
320
321 return true;
322 }
323
324 /**
325 * wx_alloc_rx_buffers - Replace used receive buffers
326 * @rx_ring: ring to place buffers on
327 * @cleaned_count: number of buffers to replace
328 **/
wx_alloc_rx_buffers(struct wx_ring * rx_ring,u16 cleaned_count)329 void wx_alloc_rx_buffers(struct wx_ring *rx_ring, u16 cleaned_count)
330 {
331 u16 i = rx_ring->next_to_use;
332 union wx_rx_desc *rx_desc;
333 struct wx_rx_buffer *bi;
334
335 /* nothing to do */
336 if (!cleaned_count)
337 return;
338
339 rx_desc = WX_RX_DESC(rx_ring, i);
340 bi = &rx_ring->rx_buffer_info[i];
341 i -= rx_ring->count;
342
343 do {
344 if (!wx_alloc_mapped_page(rx_ring, bi))
345 break;
346
347 /* sync the buffer for use by the device */
348 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
349 bi->page_offset,
350 WX_RX_BUFSZ,
351 DMA_FROM_DEVICE);
352
353 rx_desc->read.pkt_addr =
354 cpu_to_le64(bi->page_dma + bi->page_offset);
355
356 rx_desc++;
357 bi++;
358 i++;
359 if (unlikely(!i)) {
360 rx_desc = WX_RX_DESC(rx_ring, 0);
361 bi = rx_ring->rx_buffer_info;
362 i -= rx_ring->count;
363 }
364
365 /* clear the status bits for the next_to_use descriptor */
366 rx_desc->wb.upper.status_error = 0;
367
368 cleaned_count--;
369 } while (cleaned_count);
370
371 i += rx_ring->count;
372
373 if (rx_ring->next_to_use != i) {
374 rx_ring->next_to_use = i;
375 /* update next to alloc since we have filled the ring */
376 rx_ring->next_to_alloc = i;
377
378 /* Force memory writes to complete before letting h/w
379 * know there are new descriptors to fetch. (Only
380 * applicable for weak-ordered memory model archs,
381 * such as IA-64).
382 */
383 wmb();
384 writel(i, rx_ring->tail);
385 }
386 }
387
wx_desc_unused(struct wx_ring * ring)388 u16 wx_desc_unused(struct wx_ring *ring)
389 {
390 u16 ntc = ring->next_to_clean;
391 u16 ntu = ring->next_to_use;
392
393 return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1;
394 }
395
396 /**
397 * wx_is_non_eop - process handling of non-EOP buffers
398 * @rx_ring: Rx ring being processed
399 * @rx_desc: Rx descriptor for current buffer
400 * @skb: Current socket buffer containing buffer in progress
401 *
402 * This function updates next to clean. If the buffer is an EOP buffer
403 * this function exits returning false, otherwise it will place the
404 * sk_buff in the next buffer to be chained and return true indicating
405 * that this is in fact a non-EOP buffer.
406 **/
wx_is_non_eop(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)407 static bool wx_is_non_eop(struct wx_ring *rx_ring,
408 union wx_rx_desc *rx_desc,
409 struct sk_buff *skb)
410 {
411 u32 ntc = rx_ring->next_to_clean + 1;
412
413 /* fetch, update, and store next to clean */
414 ntc = (ntc < rx_ring->count) ? ntc : 0;
415 rx_ring->next_to_clean = ntc;
416
417 prefetch(WX_RX_DESC(rx_ring, ntc));
418
419 /* if we are the last buffer then there is nothing else to do */
420 if (likely(wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)))
421 return false;
422
423 rx_ring->rx_buffer_info[ntc].skb = skb;
424 rx_ring->rx_stats.non_eop_descs++;
425
426 return true;
427 }
428
wx_pull_tail(struct sk_buff * skb)429 static void wx_pull_tail(struct sk_buff *skb)
430 {
431 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
432 unsigned int pull_len;
433 unsigned char *va;
434
435 /* it is valid to use page_address instead of kmap since we are
436 * working with pages allocated out of the lomem pool per
437 * alloc_page(GFP_ATOMIC)
438 */
439 va = skb_frag_address(frag);
440
441 /* we need the header to contain the greater of either ETH_HLEN or
442 * 60 bytes if the skb->len is less than 60 for skb_pad.
443 */
444 pull_len = eth_get_headlen(skb->dev, va, WX_RXBUFFER_256);
445
446 /* align pull length to size of long to optimize memcpy performance */
447 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
448
449 /* update all of the pointers */
450 skb_frag_size_sub(frag, pull_len);
451 skb_frag_off_add(frag, pull_len);
452 skb->data_len -= pull_len;
453 skb->tail += pull_len;
454 }
455
456 /**
457 * wx_cleanup_headers - Correct corrupted or empty headers
458 * @rx_ring: rx descriptor ring packet is being transacted on
459 * @rx_desc: pointer to the EOP Rx descriptor
460 * @skb: pointer to current skb being fixed
461 *
462 * Check for corrupted packet headers caused by senders on the local L2
463 * embedded NIC switch not setting up their Tx Descriptors right. These
464 * should be very rare.
465 *
466 * Also address the case where we are pulling data in on pages only
467 * and as such no data is present in the skb header.
468 *
469 * In addition if skb is not at least 60 bytes we need to pad it so that
470 * it is large enough to qualify as a valid Ethernet frame.
471 *
472 * Returns true if an error was encountered and skb was freed.
473 **/
wx_cleanup_headers(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)474 static bool wx_cleanup_headers(struct wx_ring *rx_ring,
475 union wx_rx_desc *rx_desc,
476 struct sk_buff *skb)
477 {
478 struct net_device *netdev = rx_ring->netdev;
479
480 /* verify that the packet does not have any known errors */
481 if (!netdev ||
482 unlikely(wx_test_staterr(rx_desc, WX_RXD_ERR_RXE) &&
483 !(netdev->features & NETIF_F_RXALL))) {
484 dev_kfree_skb_any(skb);
485 return true;
486 }
487
488 /* place header in linear portion of buffer */
489 if (!skb_headlen(skb))
490 wx_pull_tail(skb);
491
492 /* if eth_skb_pad returns an error the skb was freed */
493 if (eth_skb_pad(skb))
494 return true;
495
496 return false;
497 }
498
wx_rx_hash(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)499 static void wx_rx_hash(struct wx_ring *ring,
500 union wx_rx_desc *rx_desc,
501 struct sk_buff *skb)
502 {
503 u16 rss_type;
504
505 if (!(ring->netdev->features & NETIF_F_RXHASH))
506 return;
507
508 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
509 WX_RXD_RSSTYPE_MASK;
510
511 if (!rss_type)
512 return;
513
514 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
515 (WX_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
516 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
517 }
518
519 /**
520 * wx_rx_checksum - indicate in skb if hw indicated a good cksum
521 * @ring: structure containing ring specific data
522 * @rx_desc: current Rx descriptor being processed
523 * @skb: skb currently being received and modified
524 **/
wx_rx_checksum(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)525 static void wx_rx_checksum(struct wx_ring *ring,
526 union wx_rx_desc *rx_desc,
527 struct sk_buff *skb)
528 {
529 struct wx_dec_ptype dptype = wx_decode_ptype(WX_RXD_PKTTYPE(rx_desc));
530
531 skb_checksum_none_assert(skb);
532 /* Rx csum disabled */
533 if (!(ring->netdev->features & NETIF_F_RXCSUM))
534 return;
535
536 /* if IPv4 header checksum error */
537 if ((wx_test_staterr(rx_desc, WX_RXD_STAT_IPCS) &&
538 wx_test_staterr(rx_desc, WX_RXD_ERR_IPE)) ||
539 (wx_test_staterr(rx_desc, WX_RXD_STAT_OUTERIPCS) &&
540 wx_test_staterr(rx_desc, WX_RXD_ERR_OUTERIPER))) {
541 ring->rx_stats.csum_err++;
542 return;
543 }
544
545 /* L4 checksum offload flag must set for the below code to work */
546 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_L4CS))
547 return;
548
549 /* Hardware can't guarantee csum if IPv6 Dest Header found */
550 if (dptype.prot != WX_DEC_PTYPE_PROT_SCTP &&
551 wx_test_staterr(rx_desc, WX_RXD_STAT_IPV6EX))
552 return;
553
554 /* if L4 checksum error */
555 if (wx_test_staterr(rx_desc, WX_RXD_ERR_TCPE)) {
556 ring->rx_stats.csum_err++;
557 return;
558 }
559
560 /* It must be a TCP or UDP or SCTP packet with a valid checksum */
561 skb->ip_summed = CHECKSUM_UNNECESSARY;
562
563 /* If there is an outer header present that might contain a checksum
564 * we need to bump the checksum level by 1 to reflect the fact that
565 * we are indicating we validated the inner checksum.
566 */
567 if (dptype.etype >= WX_DEC_PTYPE_ETYPE_IG)
568 __skb_incr_checksum_unnecessary(skb);
569 ring->rx_stats.csum_good_cnt++;
570 }
571
wx_rx_vlan(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)572 static void wx_rx_vlan(struct wx_ring *ring, union wx_rx_desc *rx_desc,
573 struct sk_buff *skb)
574 {
575 u16 ethertype;
576 u8 idx = 0;
577
578 if ((ring->netdev->features &
579 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) &&
580 wx_test_staterr(rx_desc, WX_RXD_STAT_VP)) {
581 idx = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
582 0x1c0) >> 6;
583 ethertype = ring->q_vector->wx->tpid[idx];
584 __vlan_hwaccel_put_tag(skb, htons(ethertype),
585 le16_to_cpu(rx_desc->wb.upper.vlan));
586 }
587 }
588
589 /**
590 * wx_process_skb_fields - Populate skb header fields from Rx descriptor
591 * @rx_ring: rx descriptor ring packet is being transacted on
592 * @rx_desc: pointer to the EOP Rx descriptor
593 * @skb: pointer to current skb being populated
594 *
595 * This function checks the ring, descriptor, and packet information in
596 * order to populate the hash, checksum, protocol, and
597 * other fields within the skb.
598 **/
wx_process_skb_fields(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)599 static void wx_process_skb_fields(struct wx_ring *rx_ring,
600 union wx_rx_desc *rx_desc,
601 struct sk_buff *skb)
602 {
603 struct wx *wx = netdev_priv(rx_ring->netdev);
604
605 wx_rx_hash(rx_ring, rx_desc, skb);
606 wx_rx_checksum(rx_ring, rx_desc, skb);
607
608 if (unlikely(test_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, wx->flags)) &&
609 unlikely(wx_test_staterr(rx_desc, WX_RXD_STAT_TS))) {
610 wx_ptp_rx_hwtstamp(rx_ring->q_vector->wx, skb);
611 rx_ring->last_rx_timestamp = jiffies;
612 }
613
614 wx_rx_vlan(rx_ring, rx_desc, skb);
615 skb_record_rx_queue(skb, rx_ring->queue_index);
616 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
617 }
618
619 /**
620 * wx_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
621 * @q_vector: structure containing interrupt and ring information
622 * @rx_ring: rx descriptor ring to transact packets on
623 * @budget: Total limit on number of packets to process
624 *
625 * This function provides a "bounce buffer" approach to Rx interrupt
626 * processing. The advantage to this is that on systems that have
627 * expensive overhead for IOMMU access this provides a means of avoiding
628 * it by maintaining the mapping of the page to the system.
629 *
630 * Returns amount of work completed.
631 **/
wx_clean_rx_irq(struct wx_q_vector * q_vector,struct wx_ring * rx_ring,int budget)632 static int wx_clean_rx_irq(struct wx_q_vector *q_vector,
633 struct wx_ring *rx_ring,
634 int budget)
635 {
636 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
637 u16 cleaned_count = wx_desc_unused(rx_ring);
638
639 do {
640 struct wx_rx_buffer *rx_buffer;
641 union wx_rx_desc *rx_desc;
642 struct sk_buff *skb;
643 int rx_buffer_pgcnt;
644
645 /* return some buffers to hardware, one at a time is too slow */
646 if (cleaned_count >= WX_RX_BUFFER_WRITE) {
647 wx_alloc_rx_buffers(rx_ring, cleaned_count);
648 cleaned_count = 0;
649 }
650
651 rx_desc = WX_RX_DESC(rx_ring, rx_ring->next_to_clean);
652 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_DD))
653 break;
654
655 /* This memory barrier is needed to keep us from reading
656 * any other fields out of the rx_desc until we know the
657 * descriptor has been written back
658 */
659 dma_rmb();
660
661 rx_buffer = wx_get_rx_buffer(rx_ring, rx_desc, &skb, &rx_buffer_pgcnt);
662
663 /* retrieve a buffer from the ring */
664 skb = wx_build_skb(rx_ring, rx_buffer, rx_desc);
665
666 /* exit if we failed to retrieve a buffer */
667 if (!skb) {
668 rx_ring->rx_stats.alloc_rx_buff_failed++;
669 break;
670 }
671
672 wx_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt);
673 cleaned_count++;
674
675 /* place incomplete frames back on ring for completion */
676 if (wx_is_non_eop(rx_ring, rx_desc, skb))
677 continue;
678
679 /* verify the packet layout is correct */
680 if (wx_cleanup_headers(rx_ring, rx_desc, skb))
681 continue;
682
683 /* probably a little skewed due to removing CRC */
684 total_rx_bytes += skb->len;
685
686 /* populate checksum, timestamp, VLAN, and protocol */
687 wx_process_skb_fields(rx_ring, rx_desc, skb);
688 napi_gro_receive(&q_vector->napi, skb);
689
690 /* update budget accounting */
691 total_rx_packets++;
692 } while (likely(total_rx_packets < budget));
693
694 u64_stats_update_begin(&rx_ring->syncp);
695 rx_ring->stats.packets += total_rx_packets;
696 rx_ring->stats.bytes += total_rx_bytes;
697 u64_stats_update_end(&rx_ring->syncp);
698 q_vector->rx.total_packets += total_rx_packets;
699 q_vector->rx.total_bytes += total_rx_bytes;
700
701 return total_rx_packets;
702 }
703
wx_txring_txq(const struct wx_ring * ring)704 static struct netdev_queue *wx_txring_txq(const struct wx_ring *ring)
705 {
706 return netdev_get_tx_queue(ring->netdev, ring->queue_index);
707 }
708
709 /**
710 * wx_clean_tx_irq - Reclaim resources after transmit completes
711 * @q_vector: structure containing interrupt and ring information
712 * @tx_ring: tx ring to clean
713 * @napi_budget: Used to determine if we are in netpoll
714 **/
wx_clean_tx_irq(struct wx_q_vector * q_vector,struct wx_ring * tx_ring,int napi_budget)715 static bool wx_clean_tx_irq(struct wx_q_vector *q_vector,
716 struct wx_ring *tx_ring, int napi_budget)
717 {
718 unsigned int budget = q_vector->wx->tx_work_limit;
719 unsigned int total_bytes = 0, total_packets = 0;
720 struct wx *wx = netdev_priv(tx_ring->netdev);
721 unsigned int i = tx_ring->next_to_clean;
722 struct wx_tx_buffer *tx_buffer;
723 union wx_tx_desc *tx_desc;
724
725 if (!netif_carrier_ok(tx_ring->netdev))
726 return true;
727
728 tx_buffer = &tx_ring->tx_buffer_info[i];
729 tx_desc = WX_TX_DESC(tx_ring, i);
730 i -= tx_ring->count;
731
732 do {
733 union wx_tx_desc *eop_desc = tx_buffer->next_to_watch;
734
735 /* if next_to_watch is not set then there is no work pending */
736 if (!eop_desc)
737 break;
738
739 /* prevent any other reads prior to eop_desc */
740 smp_rmb();
741
742 /* if DD is not set pending work has not been completed */
743 if (!(eop_desc->wb.status & cpu_to_le32(WX_TXD_STAT_DD)))
744 break;
745
746 /* clear next_to_watch to prevent false hangs */
747 tx_buffer->next_to_watch = NULL;
748
749 /* update the statistics for this packet */
750 total_bytes += tx_buffer->bytecount;
751 total_packets += tx_buffer->gso_segs;
752
753 /* schedule check for Tx timestamp */
754 if (unlikely(test_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state)) &&
755 skb_shinfo(tx_buffer->skb)->tx_flags & SKBTX_IN_PROGRESS)
756 ptp_schedule_worker(wx->ptp_clock, 0);
757
758 /* free the skb */
759 napi_consume_skb(tx_buffer->skb, napi_budget);
760
761 /* unmap skb header data */
762 dma_unmap_single(tx_ring->dev,
763 dma_unmap_addr(tx_buffer, dma),
764 dma_unmap_len(tx_buffer, len),
765 DMA_TO_DEVICE);
766
767 /* clear tx_buffer data */
768 dma_unmap_len_set(tx_buffer, len, 0);
769
770 /* unmap remaining buffers */
771 while (tx_desc != eop_desc) {
772 tx_buffer++;
773 tx_desc++;
774 i++;
775 if (unlikely(!i)) {
776 i -= tx_ring->count;
777 tx_buffer = tx_ring->tx_buffer_info;
778 tx_desc = WX_TX_DESC(tx_ring, 0);
779 }
780
781 /* unmap any remaining paged data */
782 if (dma_unmap_len(tx_buffer, len)) {
783 dma_unmap_page(tx_ring->dev,
784 dma_unmap_addr(tx_buffer, dma),
785 dma_unmap_len(tx_buffer, len),
786 DMA_TO_DEVICE);
787 dma_unmap_len_set(tx_buffer, len, 0);
788 }
789 }
790
791 /* move us one more past the eop_desc for start of next pkt */
792 tx_buffer++;
793 tx_desc++;
794 i++;
795 if (unlikely(!i)) {
796 i -= tx_ring->count;
797 tx_buffer = tx_ring->tx_buffer_info;
798 tx_desc = WX_TX_DESC(tx_ring, 0);
799 }
800
801 /* issue prefetch for next Tx descriptor */
802 prefetch(tx_desc);
803
804 /* update budget accounting */
805 budget--;
806 } while (likely(budget));
807
808 i += tx_ring->count;
809 tx_ring->next_to_clean = i;
810 u64_stats_update_begin(&tx_ring->syncp);
811 tx_ring->stats.bytes += total_bytes;
812 tx_ring->stats.packets += total_packets;
813 u64_stats_update_end(&tx_ring->syncp);
814 q_vector->tx.total_bytes += total_bytes;
815 q_vector->tx.total_packets += total_packets;
816
817 netdev_tx_completed_queue(wx_txring_txq(tx_ring),
818 total_packets, total_bytes);
819
820 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
821 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
822 (wx_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
823 /* Make sure that anybody stopping the queue after this
824 * sees the new next_to_clean.
825 */
826 smp_mb();
827
828 if (__netif_subqueue_stopped(tx_ring->netdev,
829 tx_ring->queue_index) &&
830 netif_running(tx_ring->netdev)) {
831 netif_wake_subqueue(tx_ring->netdev,
832 tx_ring->queue_index);
833 ++tx_ring->tx_stats.restart_queue;
834 }
835 }
836
837 return !!budget;
838 }
839
840 /**
841 * wx_poll - NAPI polling RX/TX cleanup routine
842 * @napi: napi struct with our devices info in it
843 * @budget: amount of work driver is allowed to do this pass, in packets
844 *
845 * This function will clean all queues associated with a q_vector.
846 **/
wx_poll(struct napi_struct * napi,int budget)847 static int wx_poll(struct napi_struct *napi, int budget)
848 {
849 struct wx_q_vector *q_vector = container_of(napi, struct wx_q_vector, napi);
850 int per_ring_budget, work_done = 0;
851 struct wx *wx = q_vector->wx;
852 bool clean_complete = true;
853 struct wx_ring *ring;
854
855 wx_for_each_ring(ring, q_vector->tx) {
856 if (!wx_clean_tx_irq(q_vector, ring, budget))
857 clean_complete = false;
858 }
859
860 /* Exit if we are called by netpoll */
861 if (budget <= 0)
862 return budget;
863
864 /* attempt to distribute budget to each queue fairly, but don't allow
865 * the budget to go below 1 because we'll exit polling
866 */
867 if (q_vector->rx.count > 1)
868 per_ring_budget = max(budget / q_vector->rx.count, 1);
869 else
870 per_ring_budget = budget;
871
872 wx_for_each_ring(ring, q_vector->rx) {
873 int cleaned = wx_clean_rx_irq(q_vector, ring, per_ring_budget);
874
875 work_done += cleaned;
876 if (cleaned >= per_ring_budget)
877 clean_complete = false;
878 }
879
880 /* If all work not completed, return budget and keep polling */
881 if (!clean_complete)
882 return budget;
883
884 /* all work done, exit the polling mode */
885 if (likely(napi_complete_done(napi, work_done))) {
886 if (netif_running(wx->netdev))
887 wx_intr_enable(wx, WX_INTR_Q(q_vector->v_idx));
888 }
889
890 return min(work_done, budget - 1);
891 }
892
wx_maybe_stop_tx(struct wx_ring * tx_ring,u16 size)893 static int wx_maybe_stop_tx(struct wx_ring *tx_ring, u16 size)
894 {
895 if (likely(wx_desc_unused(tx_ring) >= size))
896 return 0;
897
898 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
899
900 /* For the next check */
901 smp_mb();
902
903 /* We need to check again in a case another CPU has just
904 * made room available.
905 */
906 if (likely(wx_desc_unused(tx_ring) < size))
907 return -EBUSY;
908
909 /* A reprieve! - use start_queue because it doesn't call schedule */
910 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
911 ++tx_ring->tx_stats.restart_queue;
912
913 return 0;
914 }
915
wx_tx_cmd_type(u32 tx_flags)916 static u32 wx_tx_cmd_type(u32 tx_flags)
917 {
918 /* set type for advanced descriptor with frame checksum insertion */
919 u32 cmd_type = WX_TXD_DTYP_DATA | WX_TXD_IFCS;
920
921 /* set HW vlan bit if vlan is present */
922 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_HW_VLAN, WX_TXD_VLE);
923 /* set segmentation enable bits for TSO/FSO */
924 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSO, WX_TXD_TSE);
925 /* set timestamp bit if present */
926 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSTAMP, WX_TXD_MAC_TSTAMP);
927 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_LINKSEC, WX_TXD_LINKSEC);
928
929 return cmd_type;
930 }
931
wx_tx_olinfo_status(union wx_tx_desc * tx_desc,u32 tx_flags,unsigned int paylen)932 static void wx_tx_olinfo_status(union wx_tx_desc *tx_desc,
933 u32 tx_flags, unsigned int paylen)
934 {
935 u32 olinfo_status = paylen << WX_TXD_PAYLEN_SHIFT;
936
937 /* enable L4 checksum for TSO and TX checksum offload */
938 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CSUM, WX_TXD_L4CS);
939 /* enable IPv4 checksum for TSO */
940 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPV4, WX_TXD_IIPCS);
941 /* enable outer IPv4 checksum for TSO */
942 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_OUTER_IPV4,
943 WX_TXD_EIPCS);
944 /* Check Context must be set if Tx switch is enabled, which it
945 * always is for case where virtual functions are running
946 */
947 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CC, WX_TXD_CC);
948 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPSEC,
949 WX_TXD_IPSEC);
950 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
951 }
952
wx_tx_map(struct wx_ring * tx_ring,struct wx_tx_buffer * first,const u8 hdr_len)953 static int wx_tx_map(struct wx_ring *tx_ring,
954 struct wx_tx_buffer *first,
955 const u8 hdr_len)
956 {
957 struct sk_buff *skb = first->skb;
958 struct wx_tx_buffer *tx_buffer;
959 u32 tx_flags = first->tx_flags;
960 u16 i = tx_ring->next_to_use;
961 unsigned int data_len, size;
962 union wx_tx_desc *tx_desc;
963 skb_frag_t *frag;
964 dma_addr_t dma;
965 u32 cmd_type;
966
967 cmd_type = wx_tx_cmd_type(tx_flags);
968 tx_desc = WX_TX_DESC(tx_ring, i);
969 wx_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
970
971 size = skb_headlen(skb);
972 data_len = skb->data_len;
973 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
974
975 tx_buffer = first;
976
977 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
978 if (dma_mapping_error(tx_ring->dev, dma))
979 goto dma_error;
980
981 /* record length, and DMA address */
982 dma_unmap_len_set(tx_buffer, len, size);
983 dma_unmap_addr_set(tx_buffer, dma, dma);
984
985 tx_desc->read.buffer_addr = cpu_to_le64(dma);
986
987 while (unlikely(size > WX_MAX_DATA_PER_TXD)) {
988 tx_desc->read.cmd_type_len =
989 cpu_to_le32(cmd_type ^ WX_MAX_DATA_PER_TXD);
990
991 i++;
992 tx_desc++;
993 if (i == tx_ring->count) {
994 tx_desc = WX_TX_DESC(tx_ring, 0);
995 i = 0;
996 }
997 tx_desc->read.olinfo_status = 0;
998
999 dma += WX_MAX_DATA_PER_TXD;
1000 size -= WX_MAX_DATA_PER_TXD;
1001
1002 tx_desc->read.buffer_addr = cpu_to_le64(dma);
1003 }
1004
1005 if (likely(!data_len))
1006 break;
1007
1008 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
1009
1010 i++;
1011 tx_desc++;
1012 if (i == tx_ring->count) {
1013 tx_desc = WX_TX_DESC(tx_ring, 0);
1014 i = 0;
1015 }
1016 tx_desc->read.olinfo_status = 0;
1017
1018 size = skb_frag_size(frag);
1019
1020 data_len -= size;
1021
1022 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
1023 DMA_TO_DEVICE);
1024
1025 tx_buffer = &tx_ring->tx_buffer_info[i];
1026 }
1027
1028 /* write last descriptor with RS and EOP bits */
1029 cmd_type |= size | WX_TXD_EOP | WX_TXD_RS;
1030 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1031
1032 netdev_tx_sent_queue(wx_txring_txq(tx_ring), first->bytecount);
1033
1034 /* set the timestamp */
1035 first->time_stamp = jiffies;
1036 skb_tx_timestamp(skb);
1037
1038 /* Force memory writes to complete before letting h/w know there
1039 * are new descriptors to fetch. (Only applicable for weak-ordered
1040 * memory model archs, such as IA-64).
1041 *
1042 * We also need this memory barrier to make certain all of the
1043 * status bits have been updated before next_to_watch is written.
1044 */
1045 wmb();
1046
1047 /* set next_to_watch value indicating a packet is present */
1048 first->next_to_watch = tx_desc;
1049
1050 i++;
1051 if (i == tx_ring->count)
1052 i = 0;
1053
1054 tx_ring->next_to_use = i;
1055
1056 wx_maybe_stop_tx(tx_ring, DESC_NEEDED);
1057
1058 if (netif_xmit_stopped(wx_txring_txq(tx_ring)) || !netdev_xmit_more())
1059 writel(i, tx_ring->tail);
1060
1061 return 0;
1062 dma_error:
1063 dev_err(tx_ring->dev, "TX DMA map failed\n");
1064
1065 /* clear dma mappings for failed tx_buffer_info map */
1066 for (;;) {
1067 tx_buffer = &tx_ring->tx_buffer_info[i];
1068 if (dma_unmap_len(tx_buffer, len))
1069 dma_unmap_page(tx_ring->dev,
1070 dma_unmap_addr(tx_buffer, dma),
1071 dma_unmap_len(tx_buffer, len),
1072 DMA_TO_DEVICE);
1073 dma_unmap_len_set(tx_buffer, len, 0);
1074 if (tx_buffer == first)
1075 break;
1076 if (i == 0)
1077 i += tx_ring->count;
1078 i--;
1079 }
1080
1081 dev_kfree_skb_any(first->skb);
1082 first->skb = NULL;
1083
1084 tx_ring->next_to_use = i;
1085
1086 return -ENOMEM;
1087 }
1088
wx_tx_ctxtdesc(struct wx_ring * tx_ring,u32 vlan_macip_lens,u32 fcoe_sof_eof,u32 type_tucmd,u32 mss_l4len_idx)1089 static void wx_tx_ctxtdesc(struct wx_ring *tx_ring, u32 vlan_macip_lens,
1090 u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)
1091 {
1092 struct wx_tx_context_desc *context_desc;
1093 u16 i = tx_ring->next_to_use;
1094
1095 context_desc = WX_TX_CTXTDESC(tx_ring, i);
1096 i++;
1097 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1098
1099 /* set bits to identify this as an advanced context descriptor */
1100 type_tucmd |= WX_TXD_DTYP_CTXT;
1101 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
1102 context_desc->seqnum_seed = cpu_to_le32(fcoe_sof_eof);
1103 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
1104 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
1105 }
1106
1107 union network_header {
1108 struct iphdr *ipv4;
1109 struct ipv6hdr *ipv6;
1110 void *raw;
1111 };
1112
wx_encode_tx_desc_ptype(const struct wx_tx_buffer * first)1113 static u8 wx_encode_tx_desc_ptype(const struct wx_tx_buffer *first)
1114 {
1115 u8 tun_prot = 0, l4_prot = 0, ptype = 0;
1116 struct sk_buff *skb = first->skb;
1117 unsigned char *exthdr, *l4_hdr;
1118 __be16 frag_off;
1119
1120 if (skb->encapsulation) {
1121 union network_header hdr;
1122
1123 switch (first->protocol) {
1124 case htons(ETH_P_IP):
1125 tun_prot = ip_hdr(skb)->protocol;
1126 ptype = WX_PTYPE_TUN_IPV4;
1127 break;
1128 case htons(ETH_P_IPV6):
1129 l4_hdr = skb_transport_header(skb);
1130 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1131 tun_prot = ipv6_hdr(skb)->nexthdr;
1132 if (l4_hdr != exthdr)
1133 ipv6_skip_exthdr(skb, exthdr - skb->data, &tun_prot, &frag_off);
1134 ptype = WX_PTYPE_TUN_IPV6;
1135 break;
1136 default:
1137 return ptype;
1138 }
1139
1140 if (tun_prot == IPPROTO_IPIP || tun_prot == IPPROTO_IPV6) {
1141 hdr.raw = (void *)inner_ip_hdr(skb);
1142 ptype |= WX_PTYPE_PKT_IPIP;
1143 } else if (tun_prot == IPPROTO_UDP) {
1144 hdr.raw = (void *)inner_ip_hdr(skb);
1145 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
1146 skb->inner_protocol != htons(ETH_P_TEB)) {
1147 ptype |= WX_PTYPE_PKT_IG;
1148 } else {
1149 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1150 == htons(ETH_P_8021Q))
1151 ptype |= WX_PTYPE_PKT_IGMV;
1152 else
1153 ptype |= WX_PTYPE_PKT_IGM;
1154 }
1155
1156 } else if (tun_prot == IPPROTO_GRE) {
1157 hdr.raw = (void *)inner_ip_hdr(skb);
1158 if (skb->inner_protocol == htons(ETH_P_IP) ||
1159 skb->inner_protocol == htons(ETH_P_IPV6)) {
1160 ptype |= WX_PTYPE_PKT_IG;
1161 } else {
1162 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1163 == htons(ETH_P_8021Q))
1164 ptype |= WX_PTYPE_PKT_IGMV;
1165 else
1166 ptype |= WX_PTYPE_PKT_IGM;
1167 }
1168 } else {
1169 return ptype;
1170 }
1171
1172 switch (hdr.ipv4->version) {
1173 case IPVERSION:
1174 l4_prot = hdr.ipv4->protocol;
1175 break;
1176 case 6:
1177 l4_hdr = skb_inner_transport_header(skb);
1178 exthdr = skb_inner_network_header(skb) + sizeof(struct ipv6hdr);
1179 l4_prot = inner_ipv6_hdr(skb)->nexthdr;
1180 if (l4_hdr != exthdr)
1181 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_prot, &frag_off);
1182 ptype |= WX_PTYPE_PKT_IPV6;
1183 break;
1184 default:
1185 return ptype;
1186 }
1187 } else {
1188 switch (first->protocol) {
1189 case htons(ETH_P_IP):
1190 l4_prot = ip_hdr(skb)->protocol;
1191 ptype = WX_PTYPE_PKT_IP;
1192 break;
1193 case htons(ETH_P_IPV6):
1194 l4_hdr = skb_transport_header(skb);
1195 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1196 l4_prot = ipv6_hdr(skb)->nexthdr;
1197 if (l4_hdr != exthdr)
1198 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_prot, &frag_off);
1199 ptype = WX_PTYPE_PKT_IP | WX_PTYPE_PKT_IPV6;
1200 break;
1201 default:
1202 return WX_PTYPE_PKT_MAC | WX_PTYPE_TYP_MAC;
1203 }
1204 }
1205 switch (l4_prot) {
1206 case IPPROTO_TCP:
1207 ptype |= WX_PTYPE_TYP_TCP;
1208 break;
1209 case IPPROTO_UDP:
1210 ptype |= WX_PTYPE_TYP_UDP;
1211 break;
1212 case IPPROTO_SCTP:
1213 ptype |= WX_PTYPE_TYP_SCTP;
1214 break;
1215 default:
1216 ptype |= WX_PTYPE_TYP_IP;
1217 break;
1218 }
1219
1220 return ptype;
1221 }
1222
wx_tso(struct wx_ring * tx_ring,struct wx_tx_buffer * first,u8 * hdr_len,u8 ptype)1223 static int wx_tso(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1224 u8 *hdr_len, u8 ptype)
1225 {
1226 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1227 struct net_device *netdev = tx_ring->netdev;
1228 u32 l4len, tunhdr_eiplen_tunlen = 0;
1229 struct sk_buff *skb = first->skb;
1230 bool enc = skb->encapsulation;
1231 struct ipv6hdr *ipv6h;
1232 struct tcphdr *tcph;
1233 struct iphdr *iph;
1234 u8 tun_prot = 0;
1235 int err;
1236
1237 if (skb->ip_summed != CHECKSUM_PARTIAL)
1238 return 0;
1239
1240 if (!skb_is_gso(skb))
1241 return 0;
1242
1243 err = skb_cow_head(skb, 0);
1244 if (err < 0)
1245 return err;
1246
1247 /* indicates the inner headers in the skbuff are valid. */
1248 iph = enc ? inner_ip_hdr(skb) : ip_hdr(skb);
1249 if (iph->version == 4) {
1250 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1251 iph->tot_len = 0;
1252 iph->check = 0;
1253 tcph->check = ~csum_tcpudp_magic(iph->saddr,
1254 iph->daddr, 0,
1255 IPPROTO_TCP, 0);
1256 first->tx_flags |= WX_TX_FLAGS_TSO |
1257 WX_TX_FLAGS_CSUM |
1258 WX_TX_FLAGS_IPV4 |
1259 WX_TX_FLAGS_CC;
1260 } else if (iph->version == 6 && skb_is_gso_v6(skb)) {
1261 ipv6h = enc ? inner_ipv6_hdr(skb) : ipv6_hdr(skb);
1262 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1263 ipv6h->payload_len = 0;
1264 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr,
1265 &ipv6h->daddr, 0,
1266 IPPROTO_TCP, 0);
1267 first->tx_flags |= WX_TX_FLAGS_TSO |
1268 WX_TX_FLAGS_CSUM |
1269 WX_TX_FLAGS_CC;
1270 }
1271
1272 /* compute header lengths */
1273 l4len = enc ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1274 *hdr_len = enc ? skb_inner_transport_offset(skb) :
1275 skb_transport_offset(skb);
1276 *hdr_len += l4len;
1277
1278 /* update gso size and bytecount with header size */
1279 first->gso_segs = skb_shinfo(skb)->gso_segs;
1280 first->bytecount += (first->gso_segs - 1) * *hdr_len;
1281
1282 /* mss_l4len_id: use 0 as index for TSO */
1283 mss_l4len_idx = l4len << WX_TXD_L4LEN_SHIFT;
1284 mss_l4len_idx |= skb_shinfo(skb)->gso_size << WX_TXD_MSS_SHIFT;
1285
1286 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
1287 if (enc) {
1288 unsigned char *exthdr, *l4_hdr;
1289 __be16 frag_off;
1290
1291 switch (first->protocol) {
1292 case htons(ETH_P_IP):
1293 tun_prot = ip_hdr(skb)->protocol;
1294 first->tx_flags |= WX_TX_FLAGS_OUTER_IPV4;
1295 break;
1296 case htons(ETH_P_IPV6):
1297 l4_hdr = skb_transport_header(skb);
1298 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1299 tun_prot = ipv6_hdr(skb)->nexthdr;
1300 if (l4_hdr != exthdr)
1301 ipv6_skip_exthdr(skb, exthdr - skb->data, &tun_prot, &frag_off);
1302 break;
1303 default:
1304 break;
1305 }
1306 switch (tun_prot) {
1307 case IPPROTO_UDP:
1308 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1309 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1310 WX_TXD_OUTER_IPLEN_SHIFT) |
1311 (((skb_inner_mac_header(skb) -
1312 skb_transport_header(skb)) >> 1) <<
1313 WX_TXD_TUNNEL_LEN_SHIFT);
1314 break;
1315 case IPPROTO_GRE:
1316 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1317 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1318 WX_TXD_OUTER_IPLEN_SHIFT) |
1319 (((skb_inner_mac_header(skb) -
1320 skb_transport_header(skb)) >> 1) <<
1321 WX_TXD_TUNNEL_LEN_SHIFT);
1322 break;
1323 case IPPROTO_IPIP:
1324 case IPPROTO_IPV6:
1325 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1326 (char *)ip_hdr(skb)) >> 2) <<
1327 WX_TXD_OUTER_IPLEN_SHIFT;
1328 break;
1329 default:
1330 break;
1331 }
1332 vlan_macip_lens = skb_inner_network_header_len(skb) >> 1;
1333 } else {
1334 vlan_macip_lens = skb_network_header_len(skb) >> 1;
1335 }
1336
1337 vlan_macip_lens |= skb_network_offset(skb) << WX_TXD_MACLEN_SHIFT;
1338 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1339
1340 type_tucmd = ptype << 24;
1341 if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1342 netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1343 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1344 WX_TX_FLAGS_HW_VLAN,
1345 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1346 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1347 type_tucmd, mss_l4len_idx);
1348
1349 return 1;
1350 }
1351
wx_tx_csum(struct wx_ring * tx_ring,struct wx_tx_buffer * first,u8 ptype)1352 static void wx_tx_csum(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1353 u8 ptype)
1354 {
1355 u32 tunhdr_eiplen_tunlen = 0, vlan_macip_lens = 0;
1356 struct net_device *netdev = tx_ring->netdev;
1357 u32 mss_l4len_idx = 0, type_tucmd;
1358 struct sk_buff *skb = first->skb;
1359 u8 tun_prot = 0;
1360
1361 if (skb->ip_summed != CHECKSUM_PARTIAL) {
1362 csum_failed:
1363 if (!(first->tx_flags & WX_TX_FLAGS_HW_VLAN) &&
1364 !(first->tx_flags & WX_TX_FLAGS_CC))
1365 return;
1366 vlan_macip_lens = skb_network_offset(skb) <<
1367 WX_TXD_MACLEN_SHIFT;
1368 } else {
1369 unsigned char *exthdr, *l4_hdr;
1370 __be16 frag_off;
1371 u8 l4_prot = 0;
1372 union {
1373 struct iphdr *ipv4;
1374 struct ipv6hdr *ipv6;
1375 u8 *raw;
1376 } network_hdr;
1377 union {
1378 struct tcphdr *tcphdr;
1379 u8 *raw;
1380 } transport_hdr;
1381
1382 if (skb->encapsulation) {
1383 network_hdr.raw = skb_inner_network_header(skb);
1384 transport_hdr.raw = skb_inner_transport_header(skb);
1385 vlan_macip_lens = skb_network_offset(skb) <<
1386 WX_TXD_MACLEN_SHIFT;
1387 switch (first->protocol) {
1388 case htons(ETH_P_IP):
1389 tun_prot = ip_hdr(skb)->protocol;
1390 break;
1391 case htons(ETH_P_IPV6):
1392 l4_hdr = skb_transport_header(skb);
1393 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1394 tun_prot = ipv6_hdr(skb)->nexthdr;
1395 if (l4_hdr != exthdr)
1396 ipv6_skip_exthdr(skb, exthdr - skb->data,
1397 &tun_prot, &frag_off);
1398 break;
1399 default:
1400 return;
1401 }
1402 switch (tun_prot) {
1403 case IPPROTO_UDP:
1404 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1405 tunhdr_eiplen_tunlen |=
1406 ((skb_network_header_len(skb) >> 2) <<
1407 WX_TXD_OUTER_IPLEN_SHIFT) |
1408 (((skb_inner_mac_header(skb) -
1409 skb_transport_header(skb)) >> 1) <<
1410 WX_TXD_TUNNEL_LEN_SHIFT);
1411 break;
1412 case IPPROTO_GRE:
1413 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1414 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1415 WX_TXD_OUTER_IPLEN_SHIFT) |
1416 (((skb_inner_mac_header(skb) -
1417 skb_transport_header(skb)) >> 1) <<
1418 WX_TXD_TUNNEL_LEN_SHIFT);
1419 break;
1420 case IPPROTO_IPIP:
1421 case IPPROTO_IPV6:
1422 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1423 (char *)ip_hdr(skb)) >> 2) <<
1424 WX_TXD_OUTER_IPLEN_SHIFT;
1425 break;
1426 default:
1427 break;
1428 }
1429
1430 } else {
1431 network_hdr.raw = skb_network_header(skb);
1432 transport_hdr.raw = skb_transport_header(skb);
1433 vlan_macip_lens = skb_network_offset(skb) <<
1434 WX_TXD_MACLEN_SHIFT;
1435 }
1436
1437 switch (network_hdr.ipv4->version) {
1438 case IPVERSION:
1439 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1440 l4_prot = network_hdr.ipv4->protocol;
1441 break;
1442 case 6:
1443 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1444 exthdr = network_hdr.raw + sizeof(struct ipv6hdr);
1445 l4_prot = network_hdr.ipv6->nexthdr;
1446 if (transport_hdr.raw != exthdr)
1447 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_prot, &frag_off);
1448 break;
1449 default:
1450 break;
1451 }
1452
1453 switch (l4_prot) {
1454 case IPPROTO_TCP:
1455 mss_l4len_idx = (transport_hdr.tcphdr->doff * 4) <<
1456 WX_TXD_L4LEN_SHIFT;
1457 break;
1458 case IPPROTO_SCTP:
1459 mss_l4len_idx = sizeof(struct sctphdr) <<
1460 WX_TXD_L4LEN_SHIFT;
1461 break;
1462 case IPPROTO_UDP:
1463 mss_l4len_idx = sizeof(struct udphdr) <<
1464 WX_TXD_L4LEN_SHIFT;
1465 break;
1466 default:
1467 skb_checksum_help(skb);
1468 goto csum_failed;
1469 }
1470
1471 /* update TX checksum flag */
1472 first->tx_flags |= WX_TX_FLAGS_CSUM;
1473 }
1474 first->tx_flags |= WX_TX_FLAGS_CC;
1475 /* vlan_macip_lens: MACLEN, VLAN tag */
1476 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1477
1478 type_tucmd = ptype << 24;
1479 if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1480 netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1481 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1482 WX_TX_FLAGS_HW_VLAN,
1483 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1484 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1485 type_tucmd, mss_l4len_idx);
1486 }
1487
wx_xmit_frame_ring(struct sk_buff * skb,struct wx_ring * tx_ring)1488 static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb,
1489 struct wx_ring *tx_ring)
1490 {
1491 struct wx *wx = netdev_priv(tx_ring->netdev);
1492 u16 count = TXD_USE_COUNT(skb_headlen(skb));
1493 struct wx_tx_buffer *first;
1494 u8 hdr_len = 0, ptype;
1495 unsigned short f;
1496 u32 tx_flags = 0;
1497 int tso;
1498
1499 /* need: 1 descriptor per page * PAGE_SIZE/WX_MAX_DATA_PER_TXD,
1500 * + 1 desc for skb_headlen/WX_MAX_DATA_PER_TXD,
1501 * + 2 desc gap to keep tail from touching head,
1502 * + 1 desc for context descriptor,
1503 * otherwise try next time
1504 */
1505 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1506 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->
1507 frags[f]));
1508
1509 if (wx_maybe_stop_tx(tx_ring, count + 3)) {
1510 tx_ring->tx_stats.tx_busy++;
1511 return NETDEV_TX_BUSY;
1512 }
1513
1514 /* record the location of the first descriptor for this packet */
1515 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1516 first->skb = skb;
1517 first->bytecount = skb->len;
1518 first->gso_segs = 1;
1519
1520 /* if we have a HW VLAN tag being added default to the HW one */
1521 if (skb_vlan_tag_present(skb)) {
1522 tx_flags |= skb_vlan_tag_get(skb) << WX_TX_FLAGS_VLAN_SHIFT;
1523 tx_flags |= WX_TX_FLAGS_HW_VLAN;
1524 }
1525
1526 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1527 wx->ptp_clock) {
1528 if (wx->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
1529 !test_and_set_bit_lock(WX_STATE_PTP_TX_IN_PROGRESS,
1530 wx->state)) {
1531 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1532 tx_flags |= WX_TX_FLAGS_TSTAMP;
1533 wx->ptp_tx_skb = skb_get(skb);
1534 wx->ptp_tx_start = jiffies;
1535 } else {
1536 wx->tx_hwtstamp_skipped++;
1537 }
1538 }
1539
1540 /* record initial flags and protocol */
1541 first->tx_flags = tx_flags;
1542 first->protocol = vlan_get_protocol(skb);
1543
1544 ptype = wx_encode_tx_desc_ptype(first);
1545
1546 tso = wx_tso(tx_ring, first, &hdr_len, ptype);
1547 if (tso < 0)
1548 goto out_drop;
1549 else if (!tso)
1550 wx_tx_csum(tx_ring, first, ptype);
1551
1552 if (test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags) && tx_ring->atr_sample_rate)
1553 wx->atr(tx_ring, first, ptype);
1554
1555 if (wx_tx_map(tx_ring, first, hdr_len))
1556 goto cleanup_tx_tstamp;
1557
1558 return NETDEV_TX_OK;
1559 out_drop:
1560 dev_kfree_skb_any(first->skb);
1561 first->skb = NULL;
1562 cleanup_tx_tstamp:
1563 if (unlikely(tx_flags & WX_TX_FLAGS_TSTAMP)) {
1564 dev_kfree_skb_any(wx->ptp_tx_skb);
1565 wx->ptp_tx_skb = NULL;
1566 wx->tx_hwtstamp_errors++;
1567 clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
1568 }
1569
1570 return NETDEV_TX_OK;
1571 }
1572
wx_xmit_frame(struct sk_buff * skb,struct net_device * netdev)1573 netdev_tx_t wx_xmit_frame(struct sk_buff *skb,
1574 struct net_device *netdev)
1575 {
1576 unsigned int r_idx = skb->queue_mapping;
1577 struct wx *wx = netdev_priv(netdev);
1578 struct wx_ring *tx_ring;
1579
1580 if (!netif_carrier_ok(netdev)) {
1581 dev_kfree_skb_any(skb);
1582 return NETDEV_TX_OK;
1583 }
1584
1585 /* The minimum packet size for olinfo paylen is 17 so pad the skb
1586 * in order to meet this minimum size requirement.
1587 */
1588 if (skb_put_padto(skb, 17))
1589 return NETDEV_TX_OK;
1590
1591 if (r_idx >= wx->num_tx_queues)
1592 r_idx = r_idx % wx->num_tx_queues;
1593 tx_ring = wx->tx_ring[r_idx];
1594
1595 return wx_xmit_frame_ring(skb, tx_ring);
1596 }
1597 EXPORT_SYMBOL(wx_xmit_frame);
1598
wx_napi_enable_all(struct wx * wx)1599 void wx_napi_enable_all(struct wx *wx)
1600 {
1601 struct wx_q_vector *q_vector;
1602 int q_idx;
1603
1604 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1605 q_vector = wx->q_vector[q_idx];
1606 napi_enable(&q_vector->napi);
1607 }
1608 }
1609 EXPORT_SYMBOL(wx_napi_enable_all);
1610
wx_napi_disable_all(struct wx * wx)1611 void wx_napi_disable_all(struct wx *wx)
1612 {
1613 struct wx_q_vector *q_vector;
1614 int q_idx;
1615
1616 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1617 q_vector = wx->q_vector[q_idx];
1618 napi_disable(&q_vector->napi);
1619 }
1620 }
1621 EXPORT_SYMBOL(wx_napi_disable_all);
1622
1623 /**
1624 * wx_set_rss_queues: Allocate queues for RSS
1625 * @wx: board private structure to initialize
1626 *
1627 * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try
1628 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
1629 *
1630 **/
wx_set_rss_queues(struct wx * wx)1631 static void wx_set_rss_queues(struct wx *wx)
1632 {
1633 struct wx_ring_feature *f;
1634
1635 /* set mask for 16 queue limit of RSS */
1636 f = &wx->ring_feature[RING_F_RSS];
1637 f->indices = f->limit;
1638
1639 if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)))
1640 goto out;
1641
1642 clear_bit(WX_FLAG_FDIR_HASH, wx->flags);
1643
1644 /* Use Flow Director in addition to RSS to ensure the best
1645 * distribution of flows across cores, even when an FDIR flow
1646 * isn't matched.
1647 */
1648 if (f->indices > 1) {
1649 f = &wx->ring_feature[RING_F_FDIR];
1650
1651 f->indices = f->limit;
1652
1653 if (!(test_bit(WX_FLAG_FDIR_PERFECT, wx->flags)))
1654 set_bit(WX_FLAG_FDIR_HASH, wx->flags);
1655 }
1656
1657 out:
1658 wx->num_rx_queues = f->indices;
1659 wx->num_tx_queues = f->indices;
1660 }
1661
wx_set_num_queues(struct wx * wx)1662 static void wx_set_num_queues(struct wx *wx)
1663 {
1664 /* Start with base case */
1665 wx->num_rx_queues = 1;
1666 wx->num_tx_queues = 1;
1667 wx->queues_per_pool = 1;
1668
1669 wx_set_rss_queues(wx);
1670 }
1671
1672 /**
1673 * wx_acquire_msix_vectors - acquire MSI-X vectors
1674 * @wx: board private structure
1675 *
1676 * Attempts to acquire a suitable range of MSI-X vector interrupts. Will
1677 * return a negative error code if unable to acquire MSI-X vectors for any
1678 * reason.
1679 */
wx_acquire_msix_vectors(struct wx * wx)1680 static int wx_acquire_msix_vectors(struct wx *wx)
1681 {
1682 struct irq_affinity affd = { .pre_vectors = 1 };
1683 int nvecs, i;
1684
1685 /* We start by asking for one vector per queue pair */
1686 nvecs = max(wx->num_rx_queues, wx->num_tx_queues);
1687 nvecs = min_t(int, nvecs, num_online_cpus());
1688 nvecs = min_t(int, nvecs, wx->mac.max_msix_vectors);
1689
1690 wx->msix_q_entries = kcalloc(nvecs, sizeof(struct msix_entry),
1691 GFP_KERNEL);
1692 if (!wx->msix_q_entries)
1693 return -ENOMEM;
1694
1695 /* One for non-queue interrupts */
1696 nvecs += 1;
1697
1698 wx->msix_entry = kcalloc(1, sizeof(struct msix_entry),
1699 GFP_KERNEL);
1700 if (!wx->msix_entry) {
1701 kfree(wx->msix_q_entries);
1702 wx->msix_q_entries = NULL;
1703 return -ENOMEM;
1704 }
1705
1706 nvecs = pci_alloc_irq_vectors_affinity(wx->pdev, nvecs,
1707 nvecs,
1708 PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,
1709 &affd);
1710 if (nvecs < 0) {
1711 wx_err(wx, "Failed to allocate MSI-X interrupts. Err: %d\n", nvecs);
1712 kfree(wx->msix_q_entries);
1713 wx->msix_q_entries = NULL;
1714 kfree(wx->msix_entry);
1715 wx->msix_entry = NULL;
1716 return nvecs;
1717 }
1718
1719 wx->msix_entry->entry = 0;
1720 wx->msix_entry->vector = pci_irq_vector(wx->pdev, 0);
1721 nvecs -= 1;
1722 for (i = 0; i < nvecs; i++) {
1723 wx->msix_q_entries[i].entry = i;
1724 wx->msix_q_entries[i].vector = pci_irq_vector(wx->pdev, i + 1);
1725 }
1726
1727 wx->num_q_vectors = nvecs;
1728
1729 return 0;
1730 }
1731
1732 /**
1733 * wx_set_interrupt_capability - set MSI-X or MSI if supported
1734 * @wx: board private structure to initialize
1735 *
1736 * Attempt to configure the interrupts using the best available
1737 * capabilities of the hardware and the kernel.
1738 **/
wx_set_interrupt_capability(struct wx * wx)1739 static int wx_set_interrupt_capability(struct wx *wx)
1740 {
1741 struct pci_dev *pdev = wx->pdev;
1742 int nvecs, ret;
1743
1744 /* We will try to get MSI-X interrupts first */
1745 ret = wx_acquire_msix_vectors(wx);
1746 if (ret == 0 || (ret == -ENOMEM))
1747 return ret;
1748
1749 /* Disable RSS */
1750 dev_warn(&wx->pdev->dev, "Disabling RSS support\n");
1751 wx->ring_feature[RING_F_RSS].limit = 1;
1752
1753 wx_set_num_queues(wx);
1754
1755 /* minmum one for queue, one for misc*/
1756 nvecs = 1;
1757 nvecs = pci_alloc_irq_vectors(pdev, nvecs,
1758 nvecs, PCI_IRQ_MSI | PCI_IRQ_INTX);
1759 if (nvecs == 1) {
1760 if (pdev->msi_enabled)
1761 wx_err(wx, "Fallback to MSI.\n");
1762 else
1763 wx_err(wx, "Fallback to INTx.\n");
1764 } else {
1765 wx_err(wx, "Failed to allocate MSI/INTx interrupts. Error: %d\n", nvecs);
1766 return nvecs;
1767 }
1768
1769 pdev->irq = pci_irq_vector(pdev, 0);
1770 wx->num_q_vectors = 1;
1771
1772 return 0;
1773 }
1774
1775 /**
1776 * wx_cache_ring_rss - Descriptor ring to register mapping for RSS
1777 * @wx: board private structure to initialize
1778 *
1779 * Cache the descriptor ring offsets for RSS, ATR, FCoE, and SR-IOV.
1780 *
1781 **/
wx_cache_ring_rss(struct wx * wx)1782 static void wx_cache_ring_rss(struct wx *wx)
1783 {
1784 u16 i;
1785
1786 for (i = 0; i < wx->num_rx_queues; i++)
1787 wx->rx_ring[i]->reg_idx = i;
1788
1789 for (i = 0; i < wx->num_tx_queues; i++)
1790 wx->tx_ring[i]->reg_idx = i;
1791 }
1792
wx_add_ring(struct wx_ring * ring,struct wx_ring_container * head)1793 static void wx_add_ring(struct wx_ring *ring, struct wx_ring_container *head)
1794 {
1795 ring->next = head->ring;
1796 head->ring = ring;
1797 head->count++;
1798 }
1799
1800 /**
1801 * wx_alloc_q_vector - Allocate memory for a single interrupt vector
1802 * @wx: board private structure to initialize
1803 * @v_count: q_vectors allocated on wx, used for ring interleaving
1804 * @v_idx: index of vector in wx struct
1805 * @txr_count: total number of Tx rings to allocate
1806 * @txr_idx: index of first Tx ring to allocate
1807 * @rxr_count: total number of Rx rings to allocate
1808 * @rxr_idx: index of first Rx ring to allocate
1809 *
1810 * We allocate one q_vector. If allocation fails we return -ENOMEM.
1811 **/
wx_alloc_q_vector(struct wx * wx,unsigned int v_count,unsigned int v_idx,unsigned int txr_count,unsigned int txr_idx,unsigned int rxr_count,unsigned int rxr_idx)1812 static int wx_alloc_q_vector(struct wx *wx,
1813 unsigned int v_count, unsigned int v_idx,
1814 unsigned int txr_count, unsigned int txr_idx,
1815 unsigned int rxr_count, unsigned int rxr_idx)
1816 {
1817 struct wx_q_vector *q_vector;
1818 int ring_count, default_itr;
1819 struct wx_ring *ring;
1820
1821 /* note this will allocate space for the ring structure as well! */
1822 ring_count = txr_count + rxr_count;
1823
1824 q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
1825 GFP_KERNEL);
1826 if (!q_vector)
1827 return -ENOMEM;
1828
1829 /* initialize NAPI */
1830 netif_napi_add(wx->netdev, &q_vector->napi,
1831 wx_poll);
1832
1833 /* tie q_vector and wx together */
1834 wx->q_vector[v_idx] = q_vector;
1835 q_vector->wx = wx;
1836 q_vector->v_idx = v_idx;
1837 if (cpu_online(v_idx))
1838 q_vector->numa_node = cpu_to_node(v_idx);
1839
1840 /* initialize pointer to rings */
1841 ring = q_vector->ring;
1842
1843 switch (wx->mac.type) {
1844 case wx_mac_sp:
1845 case wx_mac_aml:
1846 default_itr = WX_12K_ITR;
1847 break;
1848 default:
1849 default_itr = WX_7K_ITR;
1850 break;
1851 }
1852
1853 /* initialize ITR */
1854 if (txr_count && !rxr_count)
1855 /* tx only vector */
1856 q_vector->itr = wx->tx_itr_setting ?
1857 default_itr : wx->tx_itr_setting;
1858 else
1859 /* rx or rx/tx vector */
1860 q_vector->itr = wx->rx_itr_setting ?
1861 default_itr : wx->rx_itr_setting;
1862
1863 while (txr_count) {
1864 /* assign generic ring traits */
1865 ring->dev = &wx->pdev->dev;
1866 ring->netdev = wx->netdev;
1867
1868 /* configure backlink on ring */
1869 ring->q_vector = q_vector;
1870
1871 /* update q_vector Tx values */
1872 wx_add_ring(ring, &q_vector->tx);
1873
1874 /* apply Tx specific ring traits */
1875 ring->count = wx->tx_ring_count;
1876
1877 ring->queue_index = txr_idx;
1878
1879 /* assign ring to wx */
1880 wx->tx_ring[txr_idx] = ring;
1881
1882 /* update count and index */
1883 txr_count--;
1884 txr_idx += v_count;
1885
1886 /* push pointer to next ring */
1887 ring++;
1888 }
1889
1890 while (rxr_count) {
1891 /* assign generic ring traits */
1892 ring->dev = &wx->pdev->dev;
1893 ring->netdev = wx->netdev;
1894
1895 /* configure backlink on ring */
1896 ring->q_vector = q_vector;
1897
1898 /* update q_vector Rx values */
1899 wx_add_ring(ring, &q_vector->rx);
1900
1901 /* apply Rx specific ring traits */
1902 ring->count = wx->rx_ring_count;
1903 ring->queue_index = rxr_idx;
1904
1905 /* assign ring to wx */
1906 wx->rx_ring[rxr_idx] = ring;
1907
1908 /* update count and index */
1909 rxr_count--;
1910 rxr_idx += v_count;
1911
1912 /* push pointer to next ring */
1913 ring++;
1914 }
1915
1916 return 0;
1917 }
1918
1919 /**
1920 * wx_free_q_vector - Free memory allocated for specific interrupt vector
1921 * @wx: board private structure to initialize
1922 * @v_idx: Index of vector to be freed
1923 *
1924 * This function frees the memory allocated to the q_vector. In addition if
1925 * NAPI is enabled it will delete any references to the NAPI struct prior
1926 * to freeing the q_vector.
1927 **/
wx_free_q_vector(struct wx * wx,int v_idx)1928 static void wx_free_q_vector(struct wx *wx, int v_idx)
1929 {
1930 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
1931 struct wx_ring *ring;
1932
1933 wx_for_each_ring(ring, q_vector->tx)
1934 wx->tx_ring[ring->queue_index] = NULL;
1935
1936 wx_for_each_ring(ring, q_vector->rx)
1937 wx->rx_ring[ring->queue_index] = NULL;
1938
1939 wx->q_vector[v_idx] = NULL;
1940 netif_napi_del(&q_vector->napi);
1941 kfree_rcu(q_vector, rcu);
1942 }
1943
1944 /**
1945 * wx_alloc_q_vectors - Allocate memory for interrupt vectors
1946 * @wx: board private structure to initialize
1947 *
1948 * We allocate one q_vector per queue interrupt. If allocation fails we
1949 * return -ENOMEM.
1950 **/
wx_alloc_q_vectors(struct wx * wx)1951 static int wx_alloc_q_vectors(struct wx *wx)
1952 {
1953 unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1954 unsigned int rxr_remaining = wx->num_rx_queues;
1955 unsigned int txr_remaining = wx->num_tx_queues;
1956 unsigned int q_vectors = wx->num_q_vectors;
1957 int rqpv, tqpv;
1958 int err;
1959
1960 for (; v_idx < q_vectors; v_idx++) {
1961 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1962 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1963 err = wx_alloc_q_vector(wx, q_vectors, v_idx,
1964 tqpv, txr_idx,
1965 rqpv, rxr_idx);
1966
1967 if (err)
1968 goto err_out;
1969
1970 /* update counts and index */
1971 rxr_remaining -= rqpv;
1972 txr_remaining -= tqpv;
1973 rxr_idx++;
1974 txr_idx++;
1975 }
1976
1977 return 0;
1978
1979 err_out:
1980 wx->num_tx_queues = 0;
1981 wx->num_rx_queues = 0;
1982 wx->num_q_vectors = 0;
1983
1984 while (v_idx--)
1985 wx_free_q_vector(wx, v_idx);
1986
1987 return -ENOMEM;
1988 }
1989
1990 /**
1991 * wx_free_q_vectors - Free memory allocated for interrupt vectors
1992 * @wx: board private structure to initialize
1993 *
1994 * This function frees the memory allocated to the q_vectors. In addition if
1995 * NAPI is enabled it will delete any references to the NAPI struct prior
1996 * to freeing the q_vector.
1997 **/
wx_free_q_vectors(struct wx * wx)1998 static void wx_free_q_vectors(struct wx *wx)
1999 {
2000 int v_idx = wx->num_q_vectors;
2001
2002 wx->num_tx_queues = 0;
2003 wx->num_rx_queues = 0;
2004 wx->num_q_vectors = 0;
2005
2006 while (v_idx--)
2007 wx_free_q_vector(wx, v_idx);
2008 }
2009
wx_reset_interrupt_capability(struct wx * wx)2010 void wx_reset_interrupt_capability(struct wx *wx)
2011 {
2012 struct pci_dev *pdev = wx->pdev;
2013
2014 if (!pdev->msi_enabled && !pdev->msix_enabled)
2015 return;
2016
2017 if (pdev->msix_enabled) {
2018 kfree(wx->msix_q_entries);
2019 wx->msix_q_entries = NULL;
2020 kfree(wx->msix_entry);
2021 wx->msix_entry = NULL;
2022 }
2023 pci_free_irq_vectors(wx->pdev);
2024 }
2025 EXPORT_SYMBOL(wx_reset_interrupt_capability);
2026
2027 /**
2028 * wx_clear_interrupt_scheme - Clear the current interrupt scheme settings
2029 * @wx: board private structure to clear interrupt scheme on
2030 *
2031 * We go through and clear interrupt specific resources and reset the structure
2032 * to pre-load conditions
2033 **/
wx_clear_interrupt_scheme(struct wx * wx)2034 void wx_clear_interrupt_scheme(struct wx *wx)
2035 {
2036 wx_free_q_vectors(wx);
2037 wx_reset_interrupt_capability(wx);
2038 }
2039 EXPORT_SYMBOL(wx_clear_interrupt_scheme);
2040
wx_init_interrupt_scheme(struct wx * wx)2041 int wx_init_interrupt_scheme(struct wx *wx)
2042 {
2043 int ret;
2044
2045 /* Number of supported queues */
2046 wx_set_num_queues(wx);
2047
2048 /* Set interrupt mode */
2049 ret = wx_set_interrupt_capability(wx);
2050 if (ret) {
2051 wx_err(wx, "Allocate irq vectors for failed.\n");
2052 return ret;
2053 }
2054
2055 /* Allocate memory for queues */
2056 ret = wx_alloc_q_vectors(wx);
2057 if (ret) {
2058 wx_err(wx, "Unable to allocate memory for queue vectors.\n");
2059 wx_reset_interrupt_capability(wx);
2060 return ret;
2061 }
2062
2063 wx_cache_ring_rss(wx);
2064
2065 return 0;
2066 }
2067 EXPORT_SYMBOL(wx_init_interrupt_scheme);
2068
wx_msix_clean_rings(int __always_unused irq,void * data)2069 irqreturn_t wx_msix_clean_rings(int __always_unused irq, void *data)
2070 {
2071 struct wx_q_vector *q_vector = data;
2072
2073 /* EIAM disabled interrupts (on this vector) for us */
2074 if (q_vector->rx.ring || q_vector->tx.ring)
2075 napi_schedule_irqoff(&q_vector->napi);
2076
2077 return IRQ_HANDLED;
2078 }
2079 EXPORT_SYMBOL(wx_msix_clean_rings);
2080
wx_free_irq(struct wx * wx)2081 void wx_free_irq(struct wx *wx)
2082 {
2083 struct pci_dev *pdev = wx->pdev;
2084 int vector;
2085
2086 if (!(pdev->msix_enabled)) {
2087 if (!wx->misc_irq_domain)
2088 free_irq(pdev->irq, wx);
2089 return;
2090 }
2091
2092 for (vector = 0; vector < wx->num_q_vectors; vector++) {
2093 struct wx_q_vector *q_vector = wx->q_vector[vector];
2094 struct msix_entry *entry = &wx->msix_q_entries[vector];
2095
2096 /* free only the irqs that were actually requested */
2097 if (!q_vector->rx.ring && !q_vector->tx.ring)
2098 continue;
2099
2100 free_irq(entry->vector, q_vector);
2101 }
2102
2103 if (!wx->misc_irq_domain)
2104 free_irq(wx->msix_entry->vector, wx);
2105 }
2106 EXPORT_SYMBOL(wx_free_irq);
2107
2108 /**
2109 * wx_setup_isb_resources - allocate interrupt status resources
2110 * @wx: board private structure
2111 *
2112 * Return 0 on success, negative on failure
2113 **/
wx_setup_isb_resources(struct wx * wx)2114 int wx_setup_isb_resources(struct wx *wx)
2115 {
2116 struct pci_dev *pdev = wx->pdev;
2117
2118 if (wx->isb_mem)
2119 return 0;
2120
2121 wx->isb_mem = dma_alloc_coherent(&pdev->dev,
2122 sizeof(u32) * 4,
2123 &wx->isb_dma,
2124 GFP_KERNEL);
2125 if (!wx->isb_mem) {
2126 wx_err(wx, "Alloc isb_mem failed\n");
2127 return -ENOMEM;
2128 }
2129
2130 return 0;
2131 }
2132 EXPORT_SYMBOL(wx_setup_isb_resources);
2133
2134 /**
2135 * wx_free_isb_resources - allocate all queues Rx resources
2136 * @wx: board private structure
2137 *
2138 * Return 0 on success, negative on failure
2139 **/
wx_free_isb_resources(struct wx * wx)2140 void wx_free_isb_resources(struct wx *wx)
2141 {
2142 struct pci_dev *pdev = wx->pdev;
2143
2144 dma_free_coherent(&pdev->dev, sizeof(u32) * 4,
2145 wx->isb_mem, wx->isb_dma);
2146 wx->isb_mem = NULL;
2147 }
2148 EXPORT_SYMBOL(wx_free_isb_resources);
2149
wx_misc_isb(struct wx * wx,enum wx_isb_idx idx)2150 u32 wx_misc_isb(struct wx *wx, enum wx_isb_idx idx)
2151 {
2152 u32 cur_tag = 0;
2153
2154 cur_tag = wx->isb_mem[WX_ISB_HEADER];
2155 wx->isb_tag[idx] = cur_tag;
2156
2157 return (__force u32)cpu_to_le32(wx->isb_mem[idx]);
2158 }
2159 EXPORT_SYMBOL(wx_misc_isb);
2160
2161 /**
2162 * wx_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
2163 * @wx: pointer to wx struct
2164 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
2165 * @queue: queue to map the corresponding interrupt to
2166 * @msix_vector: the vector to map to the corresponding queue
2167 *
2168 **/
wx_set_ivar(struct wx * wx,s8 direction,u16 queue,u16 msix_vector)2169 static void wx_set_ivar(struct wx *wx, s8 direction,
2170 u16 queue, u16 msix_vector)
2171 {
2172 u32 ivar, index;
2173
2174 if (direction == -1) {
2175 /* other causes */
2176 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2177 index = 0;
2178 ivar = rd32(wx, WX_PX_MISC_IVAR);
2179 ivar &= ~(0xFF << index);
2180 ivar |= (msix_vector << index);
2181 wr32(wx, WX_PX_MISC_IVAR, ivar);
2182 } else {
2183 /* tx or rx causes */
2184 msix_vector += 1; /* offset for queue vectors */
2185 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2186 index = ((16 * (queue & 1)) + (8 * direction));
2187 ivar = rd32(wx, WX_PX_IVAR(queue >> 1));
2188 ivar &= ~(0xFF << index);
2189 ivar |= (msix_vector << index);
2190 wr32(wx, WX_PX_IVAR(queue >> 1), ivar);
2191 }
2192 }
2193
2194 /**
2195 * wx_write_eitr - write EITR register in hardware specific way
2196 * @q_vector: structure containing interrupt and ring information
2197 *
2198 * This function is made to be called by ethtool and by the driver
2199 * when it needs to update EITR registers at runtime. Hardware
2200 * specific quirks/differences are taken care of here.
2201 */
wx_write_eitr(struct wx_q_vector * q_vector)2202 void wx_write_eitr(struct wx_q_vector *q_vector)
2203 {
2204 struct wx *wx = q_vector->wx;
2205 int v_idx = q_vector->v_idx;
2206 u32 itr_reg;
2207
2208 switch (wx->mac.type) {
2209 case wx_mac_sp:
2210 itr_reg = q_vector->itr & WX_SP_MAX_EITR;
2211 break;
2212 case wx_mac_aml:
2213 itr_reg = (q_vector->itr >> 3) & WX_AML_MAX_EITR;
2214 break;
2215 default:
2216 itr_reg = q_vector->itr & WX_EM_MAX_EITR;
2217 break;
2218 }
2219
2220 itr_reg |= WX_PX_ITR_CNT_WDIS;
2221
2222 wr32(wx, WX_PX_ITR(v_idx + 1), itr_reg);
2223 }
2224
2225 /**
2226 * wx_configure_vectors - Configure vectors for hardware
2227 * @wx: board private structure
2228 *
2229 * wx_configure_vectors sets up the hardware to properly generate MSI-X/MSI/INTx
2230 * interrupts.
2231 **/
wx_configure_vectors(struct wx * wx)2232 void wx_configure_vectors(struct wx *wx)
2233 {
2234 struct pci_dev *pdev = wx->pdev;
2235 u32 eitrsel = 0;
2236 u16 v_idx;
2237
2238 if (pdev->msix_enabled) {
2239 /* Populate MSIX to EITR Select */
2240 wr32(wx, WX_PX_ITRSEL, eitrsel);
2241 /* use EIAM to auto-mask when MSI-X interrupt is asserted
2242 * this saves a register write for every interrupt
2243 */
2244 wr32(wx, WX_PX_GPIE, WX_PX_GPIE_MODEL);
2245 } else {
2246 /* legacy interrupts, use EIAM to auto-mask when reading EICR,
2247 * specifically only auto mask tx and rx interrupts.
2248 */
2249 wr32(wx, WX_PX_GPIE, 0);
2250 }
2251
2252 /* Populate the IVAR table and set the ITR values to the
2253 * corresponding register.
2254 */
2255 for (v_idx = 0; v_idx < wx->num_q_vectors; v_idx++) {
2256 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
2257 struct wx_ring *ring;
2258
2259 wx_for_each_ring(ring, q_vector->rx)
2260 wx_set_ivar(wx, 0, ring->reg_idx, v_idx);
2261
2262 wx_for_each_ring(ring, q_vector->tx)
2263 wx_set_ivar(wx, 1, ring->reg_idx, v_idx);
2264
2265 wx_write_eitr(q_vector);
2266 }
2267
2268 wx_set_ivar(wx, -1, 0, 0);
2269 if (pdev->msix_enabled)
2270 wr32(wx, WX_PX_ITR(0), 1950);
2271 }
2272 EXPORT_SYMBOL(wx_configure_vectors);
2273
2274 /**
2275 * wx_clean_rx_ring - Free Rx Buffers per Queue
2276 * @rx_ring: ring to free buffers from
2277 **/
wx_clean_rx_ring(struct wx_ring * rx_ring)2278 static void wx_clean_rx_ring(struct wx_ring *rx_ring)
2279 {
2280 struct wx_rx_buffer *rx_buffer;
2281 u16 i = rx_ring->next_to_clean;
2282
2283 rx_buffer = &rx_ring->rx_buffer_info[i];
2284
2285 /* Free all the Rx ring sk_buffs */
2286 while (i != rx_ring->next_to_alloc) {
2287 if (rx_buffer->skb) {
2288 struct sk_buff *skb = rx_buffer->skb;
2289
2290 if (WX_CB(skb)->page_released)
2291 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
2292
2293 dev_kfree_skb(skb);
2294 }
2295
2296 /* Invalidate cache lines that may have been written to by
2297 * device so that we avoid corrupting memory.
2298 */
2299 dma_sync_single_range_for_cpu(rx_ring->dev,
2300 rx_buffer->dma,
2301 rx_buffer->page_offset,
2302 WX_RX_BUFSZ,
2303 DMA_FROM_DEVICE);
2304
2305 /* free resources associated with mapping */
2306 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
2307
2308 i++;
2309 rx_buffer++;
2310 if (i == rx_ring->count) {
2311 i = 0;
2312 rx_buffer = rx_ring->rx_buffer_info;
2313 }
2314 }
2315
2316 rx_ring->next_to_alloc = 0;
2317 rx_ring->next_to_clean = 0;
2318 rx_ring->next_to_use = 0;
2319 }
2320
2321 /**
2322 * wx_clean_all_rx_rings - Free Rx Buffers for all queues
2323 * @wx: board private structure
2324 **/
wx_clean_all_rx_rings(struct wx * wx)2325 void wx_clean_all_rx_rings(struct wx *wx)
2326 {
2327 int i;
2328
2329 for (i = 0; i < wx->num_rx_queues; i++)
2330 wx_clean_rx_ring(wx->rx_ring[i]);
2331 }
2332 EXPORT_SYMBOL(wx_clean_all_rx_rings);
2333
2334 /**
2335 * wx_free_rx_resources - Free Rx Resources
2336 * @rx_ring: ring to clean the resources from
2337 *
2338 * Free all receive software resources
2339 **/
wx_free_rx_resources(struct wx_ring * rx_ring)2340 static void wx_free_rx_resources(struct wx_ring *rx_ring)
2341 {
2342 wx_clean_rx_ring(rx_ring);
2343 kvfree(rx_ring->rx_buffer_info);
2344 rx_ring->rx_buffer_info = NULL;
2345
2346 /* if not set, then don't free */
2347 if (!rx_ring->desc)
2348 return;
2349
2350 dma_free_coherent(rx_ring->dev, rx_ring->size,
2351 rx_ring->desc, rx_ring->dma);
2352
2353 rx_ring->desc = NULL;
2354
2355 if (rx_ring->page_pool) {
2356 page_pool_destroy(rx_ring->page_pool);
2357 rx_ring->page_pool = NULL;
2358 }
2359 }
2360
2361 /**
2362 * wx_free_all_rx_resources - Free Rx Resources for All Queues
2363 * @wx: pointer to hardware structure
2364 *
2365 * Free all receive software resources
2366 **/
wx_free_all_rx_resources(struct wx * wx)2367 static void wx_free_all_rx_resources(struct wx *wx)
2368 {
2369 int i;
2370
2371 for (i = 0; i < wx->num_rx_queues; i++)
2372 wx_free_rx_resources(wx->rx_ring[i]);
2373 }
2374
2375 /**
2376 * wx_clean_tx_ring - Free Tx Buffers
2377 * @tx_ring: ring to be cleaned
2378 **/
wx_clean_tx_ring(struct wx_ring * tx_ring)2379 static void wx_clean_tx_ring(struct wx_ring *tx_ring)
2380 {
2381 struct wx_tx_buffer *tx_buffer;
2382 u16 i = tx_ring->next_to_clean;
2383
2384 tx_buffer = &tx_ring->tx_buffer_info[i];
2385
2386 while (i != tx_ring->next_to_use) {
2387 union wx_tx_desc *eop_desc, *tx_desc;
2388
2389 /* Free all the Tx ring sk_buffs */
2390 dev_kfree_skb_any(tx_buffer->skb);
2391
2392 /* unmap skb header data */
2393 dma_unmap_single(tx_ring->dev,
2394 dma_unmap_addr(tx_buffer, dma),
2395 dma_unmap_len(tx_buffer, len),
2396 DMA_TO_DEVICE);
2397
2398 /* check for eop_desc to determine the end of the packet */
2399 eop_desc = tx_buffer->next_to_watch;
2400 tx_desc = WX_TX_DESC(tx_ring, i);
2401
2402 /* unmap remaining buffers */
2403 while (tx_desc != eop_desc) {
2404 tx_buffer++;
2405 tx_desc++;
2406 i++;
2407 if (unlikely(i == tx_ring->count)) {
2408 i = 0;
2409 tx_buffer = tx_ring->tx_buffer_info;
2410 tx_desc = WX_TX_DESC(tx_ring, 0);
2411 }
2412
2413 /* unmap any remaining paged data */
2414 if (dma_unmap_len(tx_buffer, len))
2415 dma_unmap_page(tx_ring->dev,
2416 dma_unmap_addr(tx_buffer, dma),
2417 dma_unmap_len(tx_buffer, len),
2418 DMA_TO_DEVICE);
2419 }
2420
2421 /* move us one more past the eop_desc for start of next pkt */
2422 tx_buffer++;
2423 i++;
2424 if (unlikely(i == tx_ring->count)) {
2425 i = 0;
2426 tx_buffer = tx_ring->tx_buffer_info;
2427 }
2428 }
2429
2430 netdev_tx_reset_queue(wx_txring_txq(tx_ring));
2431
2432 /* reset next_to_use and next_to_clean */
2433 tx_ring->next_to_use = 0;
2434 tx_ring->next_to_clean = 0;
2435 }
2436
2437 /**
2438 * wx_clean_all_tx_rings - Free Tx Buffers for all queues
2439 * @wx: board private structure
2440 **/
wx_clean_all_tx_rings(struct wx * wx)2441 void wx_clean_all_tx_rings(struct wx *wx)
2442 {
2443 int i;
2444
2445 for (i = 0; i < wx->num_tx_queues; i++)
2446 wx_clean_tx_ring(wx->tx_ring[i]);
2447 }
2448 EXPORT_SYMBOL(wx_clean_all_tx_rings);
2449
2450 /**
2451 * wx_free_tx_resources - Free Tx Resources per Queue
2452 * @tx_ring: Tx descriptor ring for a specific queue
2453 *
2454 * Free all transmit software resources
2455 **/
wx_free_tx_resources(struct wx_ring * tx_ring)2456 static void wx_free_tx_resources(struct wx_ring *tx_ring)
2457 {
2458 wx_clean_tx_ring(tx_ring);
2459 kvfree(tx_ring->tx_buffer_info);
2460 tx_ring->tx_buffer_info = NULL;
2461
2462 /* if not set, then don't free */
2463 if (!tx_ring->desc)
2464 return;
2465
2466 dma_free_coherent(tx_ring->dev, tx_ring->size,
2467 tx_ring->desc, tx_ring->dma);
2468 tx_ring->desc = NULL;
2469 }
2470
2471 /**
2472 * wx_free_all_tx_resources - Free Tx Resources for All Queues
2473 * @wx: pointer to hardware structure
2474 *
2475 * Free all transmit software resources
2476 **/
wx_free_all_tx_resources(struct wx * wx)2477 static void wx_free_all_tx_resources(struct wx *wx)
2478 {
2479 int i;
2480
2481 for (i = 0; i < wx->num_tx_queues; i++)
2482 wx_free_tx_resources(wx->tx_ring[i]);
2483 }
2484
wx_free_resources(struct wx * wx)2485 void wx_free_resources(struct wx *wx)
2486 {
2487 wx_free_all_rx_resources(wx);
2488 wx_free_all_tx_resources(wx);
2489 }
2490 EXPORT_SYMBOL(wx_free_resources);
2491
wx_alloc_page_pool(struct wx_ring * rx_ring)2492 static int wx_alloc_page_pool(struct wx_ring *rx_ring)
2493 {
2494 int ret = 0;
2495
2496 struct page_pool_params pp_params = {
2497 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
2498 .order = 0,
2499 .pool_size = rx_ring->size,
2500 .nid = dev_to_node(rx_ring->dev),
2501 .dev = rx_ring->dev,
2502 .dma_dir = DMA_FROM_DEVICE,
2503 .offset = 0,
2504 .max_len = PAGE_SIZE,
2505 };
2506
2507 rx_ring->page_pool = page_pool_create(&pp_params);
2508 if (IS_ERR(rx_ring->page_pool)) {
2509 ret = PTR_ERR(rx_ring->page_pool);
2510 rx_ring->page_pool = NULL;
2511 }
2512
2513 return ret;
2514 }
2515
2516 /**
2517 * wx_setup_rx_resources - allocate Rx resources (Descriptors)
2518 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2519 *
2520 * Returns 0 on success, negative on failure
2521 **/
wx_setup_rx_resources(struct wx_ring * rx_ring)2522 static int wx_setup_rx_resources(struct wx_ring *rx_ring)
2523 {
2524 struct device *dev = rx_ring->dev;
2525 int orig_node = dev_to_node(dev);
2526 int numa_node = NUMA_NO_NODE;
2527 int size, ret;
2528
2529 size = sizeof(struct wx_rx_buffer) * rx_ring->count;
2530
2531 if (rx_ring->q_vector)
2532 numa_node = rx_ring->q_vector->numa_node;
2533
2534 rx_ring->rx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2535 if (!rx_ring->rx_buffer_info)
2536 rx_ring->rx_buffer_info = kvmalloc(size, GFP_KERNEL);
2537 if (!rx_ring->rx_buffer_info)
2538 goto err;
2539
2540 /* Round up to nearest 4K */
2541 rx_ring->size = rx_ring->count * sizeof(union wx_rx_desc);
2542 rx_ring->size = ALIGN(rx_ring->size, 4096);
2543
2544 set_dev_node(dev, numa_node);
2545 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2546 &rx_ring->dma, GFP_KERNEL);
2547 if (!rx_ring->desc) {
2548 set_dev_node(dev, orig_node);
2549 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2550 &rx_ring->dma, GFP_KERNEL);
2551 }
2552
2553 if (!rx_ring->desc)
2554 goto err;
2555
2556 rx_ring->next_to_clean = 0;
2557 rx_ring->next_to_use = 0;
2558
2559 ret = wx_alloc_page_pool(rx_ring);
2560 if (ret < 0) {
2561 dev_err(rx_ring->dev, "Page pool creation failed: %d\n", ret);
2562 goto err_desc;
2563 }
2564
2565 return 0;
2566
2567 err_desc:
2568 dma_free_coherent(dev, rx_ring->size, rx_ring->desc, rx_ring->dma);
2569 err:
2570 kvfree(rx_ring->rx_buffer_info);
2571 rx_ring->rx_buffer_info = NULL;
2572 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
2573 return -ENOMEM;
2574 }
2575
2576 /**
2577 * wx_setup_all_rx_resources - allocate all queues Rx resources
2578 * @wx: pointer to hardware structure
2579 *
2580 * If this function returns with an error, then it's possible one or
2581 * more of the rings is populated (while the rest are not). It is the
2582 * callers duty to clean those orphaned rings.
2583 *
2584 * Return 0 on success, negative on failure
2585 **/
wx_setup_all_rx_resources(struct wx * wx)2586 static int wx_setup_all_rx_resources(struct wx *wx)
2587 {
2588 int i, err = 0;
2589
2590 for (i = 0; i < wx->num_rx_queues; i++) {
2591 err = wx_setup_rx_resources(wx->rx_ring[i]);
2592 if (!err)
2593 continue;
2594
2595 wx_err(wx, "Allocation for Rx Queue %u failed\n", i);
2596 goto err_setup_rx;
2597 }
2598
2599 return 0;
2600 err_setup_rx:
2601 /* rewind the index freeing the rings as we go */
2602 while (i--)
2603 wx_free_rx_resources(wx->rx_ring[i]);
2604 return err;
2605 }
2606
2607 /**
2608 * wx_setup_tx_resources - allocate Tx resources (Descriptors)
2609 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2610 *
2611 * Return 0 on success, negative on failure
2612 **/
wx_setup_tx_resources(struct wx_ring * tx_ring)2613 static int wx_setup_tx_resources(struct wx_ring *tx_ring)
2614 {
2615 struct device *dev = tx_ring->dev;
2616 int orig_node = dev_to_node(dev);
2617 int numa_node = NUMA_NO_NODE;
2618 int size;
2619
2620 size = sizeof(struct wx_tx_buffer) * tx_ring->count;
2621
2622 if (tx_ring->q_vector)
2623 numa_node = tx_ring->q_vector->numa_node;
2624
2625 tx_ring->tx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2626 if (!tx_ring->tx_buffer_info)
2627 tx_ring->tx_buffer_info = kvmalloc(size, GFP_KERNEL);
2628 if (!tx_ring->tx_buffer_info)
2629 goto err;
2630
2631 /* round up to nearest 4K */
2632 tx_ring->size = tx_ring->count * sizeof(union wx_tx_desc);
2633 tx_ring->size = ALIGN(tx_ring->size, 4096);
2634
2635 set_dev_node(dev, numa_node);
2636 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
2637 &tx_ring->dma, GFP_KERNEL);
2638 if (!tx_ring->desc) {
2639 set_dev_node(dev, orig_node);
2640 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
2641 &tx_ring->dma, GFP_KERNEL);
2642 }
2643
2644 if (!tx_ring->desc)
2645 goto err;
2646
2647 tx_ring->next_to_use = 0;
2648 tx_ring->next_to_clean = 0;
2649
2650 return 0;
2651
2652 err:
2653 kvfree(tx_ring->tx_buffer_info);
2654 tx_ring->tx_buffer_info = NULL;
2655 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
2656 return -ENOMEM;
2657 }
2658
2659 /**
2660 * wx_setup_all_tx_resources - allocate all queues Tx resources
2661 * @wx: pointer to private structure
2662 *
2663 * If this function returns with an error, then it's possible one or
2664 * more of the rings is populated (while the rest are not). It is the
2665 * callers duty to clean those orphaned rings.
2666 *
2667 * Return 0 on success, negative on failure
2668 **/
wx_setup_all_tx_resources(struct wx * wx)2669 static int wx_setup_all_tx_resources(struct wx *wx)
2670 {
2671 int i, err = 0;
2672
2673 for (i = 0; i < wx->num_tx_queues; i++) {
2674 err = wx_setup_tx_resources(wx->tx_ring[i]);
2675 if (!err)
2676 continue;
2677
2678 wx_err(wx, "Allocation for Tx Queue %u failed\n", i);
2679 goto err_setup_tx;
2680 }
2681
2682 return 0;
2683 err_setup_tx:
2684 /* rewind the index freeing the rings as we go */
2685 while (i--)
2686 wx_free_tx_resources(wx->tx_ring[i]);
2687 return err;
2688 }
2689
wx_setup_resources(struct wx * wx)2690 int wx_setup_resources(struct wx *wx)
2691 {
2692 int err;
2693
2694 /* allocate transmit descriptors */
2695 err = wx_setup_all_tx_resources(wx);
2696 if (err)
2697 return err;
2698
2699 /* allocate receive descriptors */
2700 err = wx_setup_all_rx_resources(wx);
2701 if (err)
2702 goto err_free_tx;
2703
2704 err = wx_setup_isb_resources(wx);
2705 if (err)
2706 goto err_free_rx;
2707
2708 return 0;
2709
2710 err_free_rx:
2711 wx_free_all_rx_resources(wx);
2712 err_free_tx:
2713 wx_free_all_tx_resources(wx);
2714
2715 return err;
2716 }
2717 EXPORT_SYMBOL(wx_setup_resources);
2718
2719 /**
2720 * wx_get_stats64 - Get System Network Statistics
2721 * @netdev: network interface device structure
2722 * @stats: storage space for 64bit statistics
2723 */
wx_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)2724 void wx_get_stats64(struct net_device *netdev,
2725 struct rtnl_link_stats64 *stats)
2726 {
2727 struct wx *wx = netdev_priv(netdev);
2728 struct wx_hw_stats *hwstats;
2729 int i;
2730
2731 wx_update_stats(wx);
2732
2733 rcu_read_lock();
2734 for (i = 0; i < wx->num_rx_queues; i++) {
2735 struct wx_ring *ring = READ_ONCE(wx->rx_ring[i]);
2736 u64 bytes, packets;
2737 unsigned int start;
2738
2739 if (ring) {
2740 do {
2741 start = u64_stats_fetch_begin(&ring->syncp);
2742 packets = ring->stats.packets;
2743 bytes = ring->stats.bytes;
2744 } while (u64_stats_fetch_retry(&ring->syncp, start));
2745 stats->rx_packets += packets;
2746 stats->rx_bytes += bytes;
2747 }
2748 }
2749
2750 for (i = 0; i < wx->num_tx_queues; i++) {
2751 struct wx_ring *ring = READ_ONCE(wx->tx_ring[i]);
2752 u64 bytes, packets;
2753 unsigned int start;
2754
2755 if (ring) {
2756 do {
2757 start = u64_stats_fetch_begin(&ring->syncp);
2758 packets = ring->stats.packets;
2759 bytes = ring->stats.bytes;
2760 } while (u64_stats_fetch_retry(&ring->syncp,
2761 start));
2762 stats->tx_packets += packets;
2763 stats->tx_bytes += bytes;
2764 }
2765 }
2766
2767 rcu_read_unlock();
2768
2769 hwstats = &wx->stats;
2770 stats->rx_errors = hwstats->crcerrs + hwstats->rlec;
2771 stats->multicast = hwstats->qmprc;
2772 stats->rx_length_errors = hwstats->rlec;
2773 stats->rx_crc_errors = hwstats->crcerrs;
2774 }
2775 EXPORT_SYMBOL(wx_get_stats64);
2776
wx_set_features(struct net_device * netdev,netdev_features_t features)2777 int wx_set_features(struct net_device *netdev, netdev_features_t features)
2778 {
2779 netdev_features_t changed = netdev->features ^ features;
2780 struct wx *wx = netdev_priv(netdev);
2781 bool need_reset = false;
2782
2783 if (features & NETIF_F_RXHASH) {
2784 wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN,
2785 WX_RDB_RA_CTL_RSS_EN);
2786 wx->rss_enabled = true;
2787 } else {
2788 wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN, 0);
2789 wx->rss_enabled = false;
2790 }
2791
2792 netdev->features = features;
2793
2794 if (changed & NETIF_F_HW_VLAN_CTAG_RX && wx->do_reset)
2795 wx->do_reset(netdev);
2796 else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER))
2797 wx_set_rx_mode(netdev);
2798
2799 if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)))
2800 return 0;
2801
2802 /* Check if Flow Director n-tuple support was enabled or disabled. If
2803 * the state changed, we need to reset.
2804 */
2805 switch (features & NETIF_F_NTUPLE) {
2806 case NETIF_F_NTUPLE:
2807 /* turn off ATR, enable perfect filters and reset */
2808 if (!(test_and_set_bit(WX_FLAG_FDIR_PERFECT, wx->flags)))
2809 need_reset = true;
2810
2811 clear_bit(WX_FLAG_FDIR_HASH, wx->flags);
2812 break;
2813 default:
2814 /* turn off perfect filters, enable ATR and reset */
2815 if (test_and_clear_bit(WX_FLAG_FDIR_PERFECT, wx->flags))
2816 need_reset = true;
2817
2818 /* We cannot enable ATR if RSS is disabled */
2819 if (wx->ring_feature[RING_F_RSS].limit <= 1)
2820 break;
2821
2822 set_bit(WX_FLAG_FDIR_HASH, wx->flags);
2823 break;
2824 }
2825
2826 if (need_reset && wx->do_reset)
2827 wx->do_reset(netdev);
2828
2829 return 0;
2830 }
2831 EXPORT_SYMBOL(wx_set_features);
2832
2833 #define NETIF_VLAN_STRIPPING_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \
2834 NETIF_F_HW_VLAN_STAG_RX)
2835
2836 #define NETIF_VLAN_INSERTION_FEATURES (NETIF_F_HW_VLAN_CTAG_TX | \
2837 NETIF_F_HW_VLAN_STAG_TX)
2838
2839 #define NETIF_VLAN_FILTERING_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \
2840 NETIF_F_HW_VLAN_STAG_FILTER)
2841
wx_fix_features(struct net_device * netdev,netdev_features_t features)2842 netdev_features_t wx_fix_features(struct net_device *netdev,
2843 netdev_features_t features)
2844 {
2845 netdev_features_t changed = netdev->features ^ features;
2846 struct wx *wx = netdev_priv(netdev);
2847
2848 if (changed & NETIF_VLAN_STRIPPING_FEATURES) {
2849 if ((features & NETIF_VLAN_STRIPPING_FEATURES) != NETIF_VLAN_STRIPPING_FEATURES &&
2850 (features & NETIF_VLAN_STRIPPING_FEATURES) != 0) {
2851 features &= ~NETIF_VLAN_STRIPPING_FEATURES;
2852 features |= netdev->features & NETIF_VLAN_STRIPPING_FEATURES;
2853 wx_err(wx, "802.1Q and 802.1ad VLAN stripping must be either both on or both off.");
2854 }
2855 }
2856
2857 if (changed & NETIF_VLAN_INSERTION_FEATURES) {
2858 if ((features & NETIF_VLAN_INSERTION_FEATURES) != NETIF_VLAN_INSERTION_FEATURES &&
2859 (features & NETIF_VLAN_INSERTION_FEATURES) != 0) {
2860 features &= ~NETIF_VLAN_INSERTION_FEATURES;
2861 features |= netdev->features & NETIF_VLAN_INSERTION_FEATURES;
2862 wx_err(wx, "802.1Q and 802.1ad VLAN insertion must be either both on or both off.");
2863 }
2864 }
2865
2866 if (changed & NETIF_VLAN_FILTERING_FEATURES) {
2867 if ((features & NETIF_VLAN_FILTERING_FEATURES) != NETIF_VLAN_FILTERING_FEATURES &&
2868 (features & NETIF_VLAN_FILTERING_FEATURES) != 0) {
2869 features &= ~NETIF_VLAN_FILTERING_FEATURES;
2870 features |= netdev->features & NETIF_VLAN_FILTERING_FEATURES;
2871 wx_err(wx, "802.1Q and 802.1ad VLAN filtering must be either both on or both off.");
2872 }
2873 }
2874
2875 return features;
2876 }
2877 EXPORT_SYMBOL(wx_fix_features);
2878
wx_set_ring(struct wx * wx,u32 new_tx_count,u32 new_rx_count,struct wx_ring * temp_ring)2879 void wx_set_ring(struct wx *wx, u32 new_tx_count,
2880 u32 new_rx_count, struct wx_ring *temp_ring)
2881 {
2882 int i, err = 0;
2883
2884 /* Setup new Tx resources and free the old Tx resources in that order.
2885 * We can then assign the new resources to the rings via a memcpy.
2886 * The advantage to this approach is that we are guaranteed to still
2887 * have resources even in the case of an allocation failure.
2888 */
2889 if (new_tx_count != wx->tx_ring_count) {
2890 for (i = 0; i < wx->num_tx_queues; i++) {
2891 memcpy(&temp_ring[i], wx->tx_ring[i],
2892 sizeof(struct wx_ring));
2893
2894 temp_ring[i].count = new_tx_count;
2895 err = wx_setup_tx_resources(&temp_ring[i]);
2896 if (err) {
2897 wx_err(wx, "setup new tx resources failed, keep using the old config\n");
2898 while (i) {
2899 i--;
2900 wx_free_tx_resources(&temp_ring[i]);
2901 }
2902 return;
2903 }
2904 }
2905
2906 for (i = 0; i < wx->num_tx_queues; i++) {
2907 wx_free_tx_resources(wx->tx_ring[i]);
2908
2909 memcpy(wx->tx_ring[i], &temp_ring[i],
2910 sizeof(struct wx_ring));
2911 }
2912
2913 wx->tx_ring_count = new_tx_count;
2914 }
2915
2916 /* Repeat the process for the Rx rings if needed */
2917 if (new_rx_count != wx->rx_ring_count) {
2918 for (i = 0; i < wx->num_rx_queues; i++) {
2919 memcpy(&temp_ring[i], wx->rx_ring[i],
2920 sizeof(struct wx_ring));
2921
2922 temp_ring[i].count = new_rx_count;
2923 err = wx_setup_rx_resources(&temp_ring[i]);
2924 if (err) {
2925 wx_err(wx, "setup new rx resources failed, keep using the old config\n");
2926 while (i) {
2927 i--;
2928 wx_free_rx_resources(&temp_ring[i]);
2929 }
2930 return;
2931 }
2932 }
2933
2934 for (i = 0; i < wx->num_rx_queues; i++) {
2935 wx_free_rx_resources(wx->rx_ring[i]);
2936 memcpy(wx->rx_ring[i], &temp_ring[i],
2937 sizeof(struct wx_ring));
2938 }
2939
2940 wx->rx_ring_count = new_rx_count;
2941 }
2942 }
2943 EXPORT_SYMBOL(wx_set_ring);
2944
2945 MODULE_DESCRIPTION("Common library for Wangxun(R) Ethernet drivers.");
2946 MODULE_LICENSE("GPL");
2947