1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Chelsio Communications. All rights reserved. */
3
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5
6 #include <linux/skbuff.h>
7 #include <linux/module.h>
8 #include <linux/highmem.h>
9 #include <linux/ip.h>
10 #include <net/ipv6.h>
11 #include <linux/netdevice.h>
12 #include "chcr_ktls.h"
13
14 static LIST_HEAD(uld_ctx_list);
15 static DEFINE_MUTEX(dev_mutex);
16
17 /* chcr_get_nfrags_to_send: get the remaining nfrags after start offset
18 * @skb: skb
19 * @start: start offset.
20 * @len: how much data to send after @start
21 */
chcr_get_nfrags_to_send(struct sk_buff * skb,u32 start,u32 len)22 static int chcr_get_nfrags_to_send(struct sk_buff *skb, u32 start, u32 len)
23 {
24 struct skb_shared_info *si = skb_shinfo(skb);
25 u32 frag_size, skb_linear_data_len = skb_headlen(skb);
26 u8 nfrags = 0, frag_idx = 0;
27 skb_frag_t *frag;
28
29 /* if its a linear skb then return 1 */
30 if (!skb_is_nonlinear(skb))
31 return 1;
32
33 if (unlikely(start < skb_linear_data_len)) {
34 frag_size = min(len, skb_linear_data_len - start);
35 start = 0;
36 } else {
37 start -= skb_linear_data_len;
38
39 frag = &si->frags[frag_idx];
40 frag_size = skb_frag_size(frag);
41 while (start >= frag_size) {
42 start -= frag_size;
43 frag_idx++;
44 frag = &si->frags[frag_idx];
45 frag_size = skb_frag_size(frag);
46 }
47 frag_size = min(len, skb_frag_size(frag) - start);
48 }
49 len -= frag_size;
50 nfrags++;
51
52 while (len) {
53 frag_size = min(len, skb_frag_size(&si->frags[frag_idx]));
54 len -= frag_size;
55 nfrags++;
56 frag_idx++;
57 }
58 return nfrags;
59 }
60
61 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
62 /*
63 * chcr_ktls_save_keys: calculate and save crypto keys.
64 * @tx_info - driver specific tls info.
65 * @crypto_info - tls crypto information.
66 * @direction - TX/RX direction.
67 * return - SUCCESS/FAILURE.
68 */
chcr_ktls_save_keys(struct chcr_ktls_info * tx_info,struct tls_crypto_info * crypto_info,enum tls_offload_ctx_dir direction)69 static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
70 struct tls_crypto_info *crypto_info,
71 enum tls_offload_ctx_dir direction)
72 {
73 int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
74 unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
75 struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
76 struct ktls_key_ctx *kctx = &tx_info->key_ctx;
77 struct crypto_cipher *cipher;
78 unsigned char *key, *salt;
79
80 switch (crypto_info->cipher_type) {
81 case TLS_CIPHER_AES_GCM_128:
82 info_128_gcm =
83 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
84 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
85 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
86 tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
87 mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
88 tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
89 tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);
90
91 ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
92 key = info_128_gcm->key;
93 salt = info_128_gcm->salt;
94 tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;
95
96 /* The SCMD fields used when encrypting a full TLS
97 * record. Its a one time calculation till the
98 * connection exists.
99 */
100 tx_info->scmd0_seqno_numivs =
101 SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |
102 SCMD_CIPH_AUTH_SEQ_CTRL_F |
103 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |
104 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
105 SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |
106 SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |
107 SCMD_NUM_IVS_V(1);
108
109 /* keys will be sent inline. */
110 tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;
111
112 /* The SCMD fields used when encrypting a partial TLS
113 * record (no trailer and possibly a truncated payload).
114 */
115 tx_info->scmd0_short_seqno_numivs =
116 SCMD_CIPH_AUTH_SEQ_CTRL_F |
117 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |
118 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |
119 SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);
120
121 tx_info->scmd0_short_ivgen_hdrlen =
122 tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;
123
124 break;
125
126 default:
127 pr_err("GCM: cipher type 0x%x not supported\n",
128 crypto_info->cipher_type);
129 ret = -EINVAL;
130 goto out;
131 }
132
133 key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
134 roundup(keylen, 16) + ghash_size;
135 /* Calculate the H = CIPH(K, 0 repeated 16 times).
136 * It will go in key context
137 */
138 cipher = crypto_alloc_cipher("aes", 0, 0);
139 if (IS_ERR(cipher)) {
140 ret = -ENOMEM;
141 goto out;
142 }
143
144 ret = crypto_cipher_setkey(cipher, key, keylen);
145 if (ret)
146 goto out1;
147
148 memset(ghash_h, 0, ghash_size);
149 crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
150
151 /* fill the Key context */
152 if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
153 kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
154 mac_key_size,
155 key_ctx_size >> 4);
156 } else {
157 ret = -EINVAL;
158 goto out1;
159 }
160
161 memcpy(kctx->salt, salt, tx_info->salt_size);
162 memcpy(kctx->key, key, keylen);
163 memcpy(kctx->key + keylen, ghash_h, ghash_size);
164 tx_info->key_ctx_len = key_ctx_size;
165
166 out1:
167 crypto_free_cipher(cipher);
168 out:
169 return ret;
170 }
171
172 /*
173 * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
174 * @sk - tcp socket.
175 * @tx_info - driver specific tls info.
176 * @atid - connection active tid.
177 * return - send success/failure.
178 */
chcr_ktls_act_open_req(struct sock * sk,struct chcr_ktls_info * tx_info,int atid)179 static int chcr_ktls_act_open_req(struct sock *sk,
180 struct chcr_ktls_info *tx_info,
181 int atid)
182 {
183 struct inet_sock *inet = inet_sk(sk);
184 struct cpl_t6_act_open_req *cpl6;
185 struct cpl_act_open_req *cpl;
186 struct sk_buff *skb;
187 unsigned int len;
188 int qid_atid;
189 u64 options;
190
191 len = sizeof(*cpl6);
192 skb = alloc_skb(len, GFP_KERNEL);
193 if (unlikely(!skb))
194 return -ENOMEM;
195 /* mark it a control pkt */
196 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
197
198 cpl6 = __skb_put_zero(skb, len);
199 cpl = (struct cpl_act_open_req *)cpl6;
200 INIT_TP_WR(cpl6, 0);
201 qid_atid = TID_QID_V(tx_info->rx_qid) |
202 TID_TID_V(atid);
203 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
204 cpl->local_port = inet->inet_sport;
205 cpl->peer_port = inet->inet_dport;
206 cpl->local_ip = inet->inet_rcv_saddr;
207 cpl->peer_ip = inet->inet_daddr;
208
209 /* fill first 64 bit option field. */
210 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
211 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
212 cpl->opt0 = cpu_to_be64(options);
213
214 /* next 64 bit option field. */
215 options =
216 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
217 cpl->opt2 = htonl(options);
218
219 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
220 }
221
222 #if IS_ENABLED(CONFIG_IPV6)
223 /*
224 * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
225 * @sk - tcp socket.
226 * @tx_info - driver specific tls info.
227 * @atid - connection active tid.
228 * return - send success/failure.
229 */
chcr_ktls_act_open_req6(struct sock * sk,struct chcr_ktls_info * tx_info,int atid)230 static int chcr_ktls_act_open_req6(struct sock *sk,
231 struct chcr_ktls_info *tx_info,
232 int atid)
233 {
234 struct inet_sock *inet = inet_sk(sk);
235 struct cpl_t6_act_open_req6 *cpl6;
236 struct cpl_act_open_req6 *cpl;
237 struct sk_buff *skb;
238 unsigned int len;
239 int qid_atid;
240 u64 options;
241
242 len = sizeof(*cpl6);
243 skb = alloc_skb(len, GFP_KERNEL);
244 if (unlikely(!skb))
245 return -ENOMEM;
246 /* mark it a control pkt */
247 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
248
249 cpl6 = __skb_put_zero(skb, len);
250 cpl = (struct cpl_act_open_req6 *)cpl6;
251 INIT_TP_WR(cpl6, 0);
252 qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);
253 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));
254 cpl->local_port = inet->inet_sport;
255 cpl->peer_port = inet->inet_dport;
256 cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];
257 cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];
258 cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];
259 cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];
260
261 /* first 64 bit option field. */
262 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
263 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
264 cpl->opt0 = cpu_to_be64(options);
265 /* next 64 bit option field. */
266 options =
267 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
268 cpl->opt2 = htonl(options);
269
270 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
271 }
272 #endif /* #if IS_ENABLED(CONFIG_IPV6) */
273
274 /*
275 * chcr_setup_connection: create a TCB entry so that TP will form tcp packets.
276 * @sk - tcp socket.
277 * @tx_info - driver specific tls info.
278 * return: NET_TX_OK/NET_XMIT_DROP
279 */
chcr_setup_connection(struct sock * sk,struct chcr_ktls_info * tx_info)280 static int chcr_setup_connection(struct sock *sk,
281 struct chcr_ktls_info *tx_info)
282 {
283 struct tid_info *t = &tx_info->adap->tids;
284 int atid, ret = 0;
285
286 atid = cxgb4_alloc_atid(t, tx_info);
287 if (atid == -1)
288 return -EINVAL;
289
290 tx_info->atid = atid;
291
292 if (tx_info->ip_family == AF_INET) {
293 ret = chcr_ktls_act_open_req(sk, tx_info, atid);
294 #if IS_ENABLED(CONFIG_IPV6)
295 } else {
296 ret = cxgb4_clip_get(tx_info->netdev, (const u32 *)
297 &sk->sk_v6_rcv_saddr,
298 1);
299 if (ret)
300 return ret;
301 ret = chcr_ktls_act_open_req6(sk, tx_info, atid);
302 #endif
303 }
304
305 /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
306 * success, if any other return type clear atid and return that failure.
307 */
308 if (ret) {
309 if (ret == NET_XMIT_CN) {
310 ret = 0;
311 } else {
312 #if IS_ENABLED(CONFIG_IPV6)
313 /* clear clip entry */
314 if (tx_info->ip_family == AF_INET6)
315 cxgb4_clip_release(tx_info->netdev,
316 (const u32 *)
317 &sk->sk_v6_rcv_saddr,
318 1);
319 #endif
320 cxgb4_free_atid(t, atid);
321 }
322 }
323
324 return ret;
325 }
326
327 /*
328 * chcr_set_tcb_field: update tcb fields.
329 * @tx_info - driver specific tls info.
330 * @word - TCB word.
331 * @mask - TCB word related mask.
332 * @val - TCB word related value.
333 * @no_reply - set 1 if not looking for TP response.
334 */
chcr_set_tcb_field(struct chcr_ktls_info * tx_info,u16 word,u64 mask,u64 val,int no_reply)335 static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
336 u64 mask, u64 val, int no_reply)
337 {
338 struct cpl_set_tcb_field *req;
339 struct sk_buff *skb;
340
341 skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
342 if (!skb)
343 return -ENOMEM;
344
345 req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
346 INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
347 req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
348 NO_REPLY_V(no_reply));
349 req->word_cookie = htons(TCB_WORD_V(word));
350 req->mask = cpu_to_be64(mask);
351 req->val = cpu_to_be64(val);
352
353 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
354 return cxgb4_ofld_send(tx_info->netdev, skb);
355 }
356
357 /*
358 * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE
359 * @tx_info - driver specific tls info.
360 * return: NET_TX_OK/NET_XMIT_DROP.
361 */
chcr_ktls_mark_tcb_close(struct chcr_ktls_info * tx_info)362 static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info)
363 {
364 return chcr_set_tcb_field(tx_info, TCB_T_STATE_W,
365 TCB_T_STATE_V(TCB_T_STATE_M),
366 CHCR_TCB_STATE_CLOSED, 1);
367 }
368
369 /*
370 * chcr_ktls_dev_del: call back for tls_dev_del.
371 * Remove the tid and l2t entry and close the connection.
372 * it per connection basis.
373 * @netdev - net device.
374 * @tls_cts - tls context.
375 * @direction - TX/RX crypto direction
376 */
chcr_ktls_dev_del(struct net_device * netdev,struct tls_context * tls_ctx,enum tls_offload_ctx_dir direction)377 static void chcr_ktls_dev_del(struct net_device *netdev,
378 struct tls_context *tls_ctx,
379 enum tls_offload_ctx_dir direction)
380 {
381 struct chcr_ktls_ofld_ctx_tx *tx_ctx =
382 chcr_get_ktls_tx_context(tls_ctx);
383 struct chcr_ktls_info *tx_info = tx_ctx->chcr_info;
384 struct ch_ktls_port_stats_debug *port_stats;
385
386 if (!tx_info)
387 return;
388
389 /* clear l2t entry */
390 if (tx_info->l2te)
391 cxgb4_l2t_release(tx_info->l2te);
392
393 #if IS_ENABLED(CONFIG_IPV6)
394 /* clear clip entry */
395 if (tx_info->ip_family == AF_INET6)
396 cxgb4_clip_release(netdev, (const u32 *)
397 &tx_info->sk->sk_v6_rcv_saddr,
398 1);
399 #endif
400
401 /* clear tid */
402 if (tx_info->tid != -1) {
403 /* clear tcb state and then release tid */
404 chcr_ktls_mark_tcb_close(tx_info);
405 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
406 tx_info->tid, tx_info->ip_family);
407 }
408
409 port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
410 atomic64_inc(&port_stats->ktls_tx_connection_close);
411 kvfree(tx_info);
412 tx_ctx->chcr_info = NULL;
413 /* release module refcount */
414 module_put(THIS_MODULE);
415 }
416
417 /*
418 * chcr_ktls_dev_add: call back for tls_dev_add.
419 * Create a tcb entry for TP. Also add l2t entry for the connection. And
420 * generate keys & save those keys locally.
421 * @netdev - net device.
422 * @tls_cts - tls context.
423 * @direction - TX/RX crypto direction
424 * return: SUCCESS/FAILURE.
425 */
chcr_ktls_dev_add(struct net_device * netdev,struct sock * sk,enum tls_offload_ctx_dir direction,struct tls_crypto_info * crypto_info,u32 start_offload_tcp_sn)426 static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
427 enum tls_offload_ctx_dir direction,
428 struct tls_crypto_info *crypto_info,
429 u32 start_offload_tcp_sn)
430 {
431 struct tls_context *tls_ctx = tls_get_ctx(sk);
432 struct ch_ktls_port_stats_debug *port_stats;
433 struct chcr_ktls_ofld_ctx_tx *tx_ctx;
434 struct chcr_ktls_info *tx_info;
435 struct dst_entry *dst;
436 struct adapter *adap;
437 struct port_info *pi;
438 struct neighbour *n;
439 u8 daaddr[16];
440 int ret = -1;
441
442 tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
443
444 pi = netdev_priv(netdev);
445 adap = pi->adapter;
446 port_stats = &adap->ch_ktls_stats.ktls_port[pi->port_id];
447 atomic64_inc(&port_stats->ktls_tx_connection_open);
448
449 if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
450 pr_err("not expecting for RX direction\n");
451 goto out;
452 }
453
454 if (tx_ctx->chcr_info)
455 goto out;
456
457 tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
458 if (!tx_info)
459 goto out;
460
461 tx_info->sk = sk;
462 spin_lock_init(&tx_info->lock);
463 /* initialize tid and atid to -1, 0 is a also a valid id. */
464 tx_info->tid = -1;
465 tx_info->atid = -1;
466
467 tx_info->adap = adap;
468 tx_info->netdev = netdev;
469 tx_info->first_qset = pi->first_qset;
470 tx_info->tx_chan = pi->tx_chan;
471 tx_info->smt_idx = pi->smt_idx;
472 tx_info->port_id = pi->port_id;
473 tx_info->prev_ack = 0;
474 tx_info->prev_win = 0;
475
476 tx_info->rx_qid = chcr_get_first_rx_qid(adap);
477 if (unlikely(tx_info->rx_qid < 0))
478 goto free_tx_info;
479
480 tx_info->prev_seq = start_offload_tcp_sn;
481 tx_info->tcp_start_seq_number = start_offload_tcp_sn;
482
483 /* save crypto keys */
484 ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
485 if (ret < 0)
486 goto free_tx_info;
487
488 /* get peer ip */
489 if (sk->sk_family == AF_INET) {
490 memcpy(daaddr, &sk->sk_daddr, 4);
491 tx_info->ip_family = AF_INET;
492 #if IS_ENABLED(CONFIG_IPV6)
493 } else {
494 if (!sk->sk_ipv6only &&
495 ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
496 memcpy(daaddr, &sk->sk_daddr, 4);
497 tx_info->ip_family = AF_INET;
498 } else {
499 memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);
500 tx_info->ip_family = AF_INET6;
501 }
502 #endif
503 }
504
505 /* get the l2t index */
506 dst = sk_dst_get(sk);
507 if (!dst) {
508 pr_err("DST entry not found\n");
509 goto free_tx_info;
510 }
511 n = dst_neigh_lookup(dst, daaddr);
512 if (!n || !n->dev) {
513 pr_err("neighbour not found\n");
514 dst_release(dst);
515 goto free_tx_info;
516 }
517 tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
518
519 neigh_release(n);
520 dst_release(dst);
521
522 if (!tx_info->l2te) {
523 pr_err("l2t entry not found\n");
524 goto free_tx_info;
525 }
526
527 /* Driver shouldn't be removed until any single connection exists */
528 if (!try_module_get(THIS_MODULE))
529 goto free_l2t;
530
531 init_completion(&tx_info->completion);
532 /* create a filter and call cxgb4_l2t_send to send the packet out, which
533 * will take care of updating l2t entry in hw if not already done.
534 */
535 tx_info->open_state = CH_KTLS_OPEN_PENDING;
536
537 if (chcr_setup_connection(sk, tx_info))
538 goto put_module;
539
540 /* Wait for reply */
541 wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
542 spin_lock_bh(&tx_info->lock);
543 if (tx_info->open_state) {
544 /* need to wait for hw response, can't free tx_info yet. */
545 if (tx_info->open_state == CH_KTLS_OPEN_PENDING)
546 tx_info->pending_close = true;
547 else
548 spin_unlock_bh(&tx_info->lock);
549 /* if in pending close, free the lock after the cleanup */
550 goto put_module;
551 }
552 spin_unlock_bh(&tx_info->lock);
553
554 /* initialize tcb */
555 reinit_completion(&tx_info->completion);
556 /* mark it pending for hw response */
557 tx_info->open_state = CH_KTLS_OPEN_PENDING;
558
559 if (chcr_init_tcb_fields(tx_info))
560 goto free_tid;
561
562 /* Wait for reply */
563 wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
564 spin_lock_bh(&tx_info->lock);
565 if (tx_info->open_state) {
566 /* need to wait for hw response, can't free tx_info yet. */
567 tx_info->pending_close = true;
568 /* free the lock after cleanup */
569 goto free_tid;
570 }
571 spin_unlock_bh(&tx_info->lock);
572
573 if (!cxgb4_check_l2t_valid(tx_info->l2te))
574 goto free_tid;
575
576 atomic64_inc(&port_stats->ktls_tx_ctx);
577 tx_ctx->chcr_info = tx_info;
578
579 return 0;
580
581 free_tid:
582 chcr_ktls_mark_tcb_close(tx_info);
583 #if IS_ENABLED(CONFIG_IPV6)
584 /* clear clip entry */
585 if (tx_info->ip_family == AF_INET6)
586 cxgb4_clip_release(netdev, (const u32 *)
587 &sk->sk_v6_rcv_saddr,
588 1);
589 #endif
590 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
591 tx_info->tid, tx_info->ip_family);
592
593 put_module:
594 /* release module refcount */
595 module_put(THIS_MODULE);
596 free_l2t:
597 cxgb4_l2t_release(tx_info->l2te);
598 free_tx_info:
599 if (tx_info->pending_close)
600 spin_unlock_bh(&tx_info->lock);
601 else
602 kvfree(tx_info);
603 out:
604 atomic64_inc(&port_stats->ktls_tx_connection_fail);
605 return -1;
606 }
607
608 /*
609 * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number
610 * handling.
611 * @tx_info - driver specific tls info.
612 * return: NET_TX_OK/NET_XMIT_DROP
613 */
chcr_init_tcb_fields(struct chcr_ktls_info * tx_info)614 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
615 {
616 int ret = 0;
617
618 /* set tcb in offload and bypass */
619 ret =
620 chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
621 TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
622 TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
623 if (ret)
624 return ret;
625 /* reset snd_una and snd_next fields in tcb */
626 ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
627 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
628 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
629 0, 1);
630 if (ret)
631 return ret;
632
633 /* reset send max */
634 ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
635 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
636 0, 1);
637 if (ret)
638 return ret;
639
640 /* update l2t index and request for tp reply to confirm tcb is
641 * initialised to handle tx traffic.
642 */
643 ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
644 TCB_L2T_IX_V(TCB_L2T_IX_M),
645 TCB_L2T_IX_V(tx_info->l2te->idx), 0);
646 return ret;
647 }
648
649 /*
650 * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
651 */
chcr_ktls_cpl_act_open_rpl(struct adapter * adap,unsigned char * input)652 static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap,
653 unsigned char *input)
654 {
655 const struct cpl_act_open_rpl *p = (void *)input;
656 struct chcr_ktls_info *tx_info = NULL;
657 unsigned int atid, tid, status;
658 struct tid_info *t;
659
660 tid = GET_TID(p);
661 status = AOPEN_STATUS_G(ntohl(p->atid_status));
662 atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
663
664 t = &adap->tids;
665 tx_info = lookup_atid(t, atid);
666
667 if (!tx_info || tx_info->atid != atid) {
668 pr_err("%s: incorrect tx_info or atid\n", __func__);
669 return -1;
670 }
671
672 cxgb4_free_atid(t, atid);
673 tx_info->atid = -1;
674
675 spin_lock(&tx_info->lock);
676 /* HW response is very close, finish pending cleanup */
677 if (tx_info->pending_close) {
678 spin_unlock(&tx_info->lock);
679 if (!status) {
680 /* it's a late success, tcb status is establised,
681 * mark it close.
682 */
683 chcr_ktls_mark_tcb_close(tx_info);
684 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
685 tid, tx_info->ip_family);
686 }
687 kvfree(tx_info);
688 return 0;
689 }
690
691 if (!status) {
692 tx_info->tid = tid;
693 cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
694 tx_info->open_state = CH_KTLS_OPEN_SUCCESS;
695 } else {
696 tx_info->open_state = CH_KTLS_OPEN_FAILURE;
697 }
698 spin_unlock(&tx_info->lock);
699
700 complete(&tx_info->completion);
701 return 0;
702 }
703
704 /*
705 * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
706 */
chcr_ktls_cpl_set_tcb_rpl(struct adapter * adap,unsigned char * input)707 static int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
708 {
709 const struct cpl_set_tcb_rpl *p = (void *)input;
710 struct chcr_ktls_info *tx_info = NULL;
711 struct tid_info *t;
712 u32 tid;
713
714 tid = GET_TID(p);
715
716 t = &adap->tids;
717 tx_info = lookup_tid(t, tid);
718
719 if (!tx_info || tx_info->tid != tid) {
720 pr_err("%s: incorrect tx_info or tid\n", __func__);
721 return -1;
722 }
723
724 spin_lock(&tx_info->lock);
725 if (tx_info->pending_close) {
726 spin_unlock(&tx_info->lock);
727 kvfree(tx_info);
728 return 0;
729 }
730 tx_info->open_state = false;
731 spin_unlock(&tx_info->lock);
732
733 complete(&tx_info->completion);
734 return 0;
735 }
736
__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info * tx_info,u32 tid,void * pos,u16 word,struct sge_eth_txq * q,u64 mask,u64 val,u32 reply)737 static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
738 u32 tid, void *pos, u16 word,
739 struct sge_eth_txq *q, u64 mask,
740 u64 val, u32 reply)
741 {
742 struct cpl_set_tcb_field_core *cpl;
743 struct ulptx_idata *idata;
744 struct ulp_txpkt *txpkt;
745
746 /* ULP_TXPKT */
747 txpkt = pos;
748 txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
749 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
750 ULP_TXPKT_FID_V(q->q.cntxt_id) |
751 ULP_TXPKT_RO_F);
752 txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));
753
754 /* ULPTX_IDATA sub-command */
755 idata = (struct ulptx_idata *)(txpkt + 1);
756 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
757 idata->len = htonl(sizeof(*cpl));
758 pos = idata + 1;
759
760 cpl = pos;
761 /* CPL_SET_TCB_FIELD */
762 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
763 cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
764 NO_REPLY_V(!reply));
765 cpl->word_cookie = htons(TCB_WORD_V(word));
766 cpl->mask = cpu_to_be64(mask);
767 cpl->val = cpu_to_be64(val);
768
769 /* ULPTX_NOOP */
770 idata = (struct ulptx_idata *)(cpl + 1);
771 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
772 idata->len = htonl(0);
773 pos = idata + 1;
774
775 return pos;
776 }
777
778
779 /*
780 * chcr_write_cpl_set_tcb_ulp: update tcb values.
781 * TCB is responsible to create tcp headers, so all the related values
782 * should be correctly updated.
783 * @tx_info - driver specific tls info.
784 * @q - tx queue on which packet is going out.
785 * @tid - TCB identifier.
786 * @pos - current index where should we start writing.
787 * @word - TCB word.
788 * @mask - TCB word related mask.
789 * @val - TCB word related value.
790 * @reply - set 1 if looking for TP response.
791 * return - next position to write.
792 */
chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tid,void * pos,u16 word,u64 mask,u64 val,u32 reply)793 static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
794 struct sge_eth_txq *q, u32 tid,
795 void *pos, u16 word, u64 mask,
796 u64 val, u32 reply)
797 {
798 int left = (void *)q->q.stat - pos;
799
800 if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {
801 if (!left) {
802 pos = q->q.desc;
803 } else {
804 u8 buf[48] = {0};
805
806 __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, q,
807 mask, val, reply);
808
809 return chcr_copy_to_txd(buf, &q->q, pos,
810 CHCR_SET_TCB_FIELD_LEN);
811 }
812 }
813
814 pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, q,
815 mask, val, reply);
816
817 /* check again if we are at the end of the queue */
818 if (left == CHCR_SET_TCB_FIELD_LEN)
819 pos = q->q.desc;
820
821 return pos;
822 }
823
824 /*
825 * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
826 * with updated values like tcp seq, ack, window etc.
827 * @tx_info - driver specific tls info.
828 * @q - TX queue.
829 * @tcp_seq
830 * @tcp_ack
831 * @tcp_win
832 * return: NETDEV_TX_BUSY/NET_TX_OK.
833 */
chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u64 tcp_seq,u64 tcp_ack,u64 tcp_win,bool offset)834 static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,
835 struct sge_eth_txq *q, u64 tcp_seq,
836 u64 tcp_ack, u64 tcp_win, bool offset)
837 {
838 bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));
839 struct ch_ktls_port_stats_debug *port_stats;
840 u32 len, cpl = 0, ndesc, wr_len, wr_mid = 0;
841 struct fw_ulptx_wr *wr;
842 int credits;
843 void *pos;
844
845 wr_len = sizeof(*wr);
846 /* there can be max 4 cpls, check if we have enough credits */
847 len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
848 ndesc = DIV_ROUND_UP(len, 64);
849
850 credits = chcr_txq_avail(&q->q) - ndesc;
851 if (unlikely(credits < 0)) {
852 chcr_eth_txq_stop(q);
853 return NETDEV_TX_BUSY;
854 }
855
856 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
857 chcr_eth_txq_stop(q);
858 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
859 }
860
861 pos = &q->q.desc[q->q.pidx];
862 /* make space for WR, we'll fill it later when we know all the cpls
863 * being sent out and have complete length.
864 */
865 wr = pos;
866 pos += wr_len;
867 /* update tx_max if its a re-transmit or the first wr */
868 if (first_wr || tcp_seq != tx_info->prev_seq) {
869 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
870 TCB_TX_MAX_W,
871 TCB_TX_MAX_V(TCB_TX_MAX_M),
872 TCB_TX_MAX_V(tcp_seq), 0);
873 cpl++;
874 }
875 /* reset snd una if it's a re-transmit pkt */
876 if (tcp_seq != tx_info->prev_seq || offset) {
877 /* reset snd_una */
878 port_stats =
879 &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
880 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
881 TCB_SND_UNA_RAW_W,
882 TCB_SND_UNA_RAW_V
883 (TCB_SND_UNA_RAW_M),
884 TCB_SND_UNA_RAW_V(0), 0);
885 if (tcp_seq != tx_info->prev_seq)
886 atomic64_inc(&port_stats->ktls_tx_ooo);
887 cpl++;
888 }
889 /* update ack */
890 if (first_wr || tx_info->prev_ack != tcp_ack) {
891 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
892 TCB_RCV_NXT_W,
893 TCB_RCV_NXT_V(TCB_RCV_NXT_M),
894 TCB_RCV_NXT_V(tcp_ack), 0);
895 tx_info->prev_ack = tcp_ack;
896 cpl++;
897 }
898 /* update receive window */
899 if (first_wr || tx_info->prev_win != tcp_win) {
900 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
901 TCB_RCV_WND_W,
902 TCB_RCV_WND_V(TCB_RCV_WND_M),
903 TCB_RCV_WND_V(tcp_win), 0);
904 tx_info->prev_win = tcp_win;
905 cpl++;
906 }
907
908 if (cpl) {
909 /* get the actual length */
910 len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
911 /* ULPTX wr */
912 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
913 wr->cookie = 0;
914 /* fill len in wr field */
915 wr->flowid_len16 = htonl(wr_mid |
916 FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
917
918 ndesc = DIV_ROUND_UP(len, 64);
919 chcr_txq_advance(&q->q, ndesc);
920 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
921 }
922 return 0;
923 }
924
925 /*
926 * chcr_ktls_get_tx_flits
927 * returns number of flits to be sent out, it includes key context length, WR
928 * size and skb fragments.
929 */
930 static unsigned int
chcr_ktls_get_tx_flits(u32 nr_frags,unsigned int key_ctx_len)931 chcr_ktls_get_tx_flits(u32 nr_frags, unsigned int key_ctx_len)
932 {
933 return chcr_sgl_len(nr_frags) +
934 DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8);
935 }
936
937 /*
938 * chcr_ktls_check_tcp_options: To check if there is any TCP option availbale
939 * other than timestamp.
940 * @skb - skb contains partial record..
941 * return: 1 / 0
942 */
943 static int
chcr_ktls_check_tcp_options(struct tcphdr * tcp)944 chcr_ktls_check_tcp_options(struct tcphdr *tcp)
945 {
946 int cnt, opt, optlen;
947 u_char *cp;
948
949 cp = (u_char *)(tcp + 1);
950 cnt = (tcp->doff << 2) - sizeof(struct tcphdr);
951 for (; cnt > 0; cnt -= optlen, cp += optlen) {
952 opt = cp[0];
953 if (opt == TCPOPT_EOL)
954 break;
955 if (opt == TCPOPT_NOP) {
956 optlen = 1;
957 } else {
958 if (cnt < 2)
959 break;
960 optlen = cp[1];
961 if (optlen < 2 || optlen > cnt)
962 break;
963 }
964 switch (opt) {
965 case TCPOPT_NOP:
966 break;
967 default:
968 return 1;
969 }
970 }
971 return 0;
972 }
973
974 /*
975 * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to
976 * send out separately.
977 * @tx_info - driver specific tls info.
978 * @skb - skb contains partial record..
979 * @q - TX queue.
980 * @tx_chan - channel number.
981 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
982 */
983 static int
chcr_ktls_write_tcp_options(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct sge_eth_txq * q,uint32_t tx_chan)984 chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb,
985 struct sge_eth_txq *q, uint32_t tx_chan)
986 {
987 struct fw_eth_tx_pkt_wr *wr;
988 struct cpl_tx_pkt_core *cpl;
989 u32 ctrl, iplen, maclen;
990 struct ipv6hdr *ip6;
991 unsigned int ndesc;
992 struct tcphdr *tcp;
993 int len16, pktlen;
994 struct iphdr *ip;
995 u32 wr_mid = 0;
996 int credits;
997 u8 buf[150];
998 u64 cntrl1;
999 void *pos;
1000
1001 iplen = skb_network_header_len(skb);
1002 maclen = skb_mac_header_len(skb);
1003
1004 /* packet length = eth hdr len + ip hdr len + tcp hdr len
1005 * (including options).
1006 */
1007 pktlen = skb_transport_offset(skb) + tcp_hdrlen(skb);
1008
1009 ctrl = sizeof(*cpl) + pktlen;
1010 len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16);
1011 /* check how many descriptors needed */
1012 ndesc = DIV_ROUND_UP(len16, 4);
1013
1014 credits = chcr_txq_avail(&q->q) - ndesc;
1015 if (unlikely(credits < 0)) {
1016 chcr_eth_txq_stop(q);
1017 return NETDEV_TX_BUSY;
1018 }
1019
1020 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1021 chcr_eth_txq_stop(q);
1022 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1023 }
1024
1025 pos = &q->q.desc[q->q.pidx];
1026 wr = pos;
1027
1028 /* Firmware work request header */
1029 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1030 FW_WR_IMMDLEN_V(ctrl));
1031
1032 wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1033 wr->r3 = 0;
1034
1035 cpl = (void *)(wr + 1);
1036
1037 /* CPL header */
1038 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) |
1039 TXPKT_PF_V(tx_info->adap->pf));
1040 cpl->pack = 0;
1041 cpl->len = htons(pktlen);
1042
1043 memcpy(buf, skb->data, pktlen);
1044 if (!IS_ENABLED(CONFIG_IPV6) || tx_info->ip_family == AF_INET) {
1045 /* we need to correct ip header len */
1046 ip = (struct iphdr *)(buf + maclen);
1047 ip->tot_len = htons(pktlen - maclen);
1048 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP);
1049 } else {
1050 ip6 = (struct ipv6hdr *)(buf + maclen);
1051 ip6->payload_len = htons(pktlen - maclen - iplen);
1052 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP6);
1053 }
1054
1055 cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1056 TXPKT_IPHDR_LEN_V(iplen);
1057 /* checksum offload */
1058 cpl->ctrl1 = cpu_to_be64(cntrl1);
1059
1060 pos = cpl + 1;
1061
1062 /* now take care of the tcp header, if fin is not set then clear push
1063 * bit as well, and if fin is set, it will be sent at the last so we
1064 * need to update the tcp sequence number as per the last packet.
1065 */
1066 tcp = (struct tcphdr *)(buf + maclen + iplen);
1067
1068 if (!tcp->fin)
1069 tcp->psh = 0;
1070 else
1071 tcp->seq = htonl(tx_info->prev_seq);
1072
1073 chcr_copy_to_txd(buf, &q->q, pos, pktlen);
1074
1075 chcr_txq_advance(&q->q, ndesc);
1076 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1077 return 0;
1078 }
1079
1080 /*
1081 * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb
1082 * received has partial end part of the record, send out the complete record, so
1083 * that crypto block will be able to generate TAG/HASH.
1084 * @skb - segment which has complete or partial end part.
1085 * @tx_info - driver specific tls info.
1086 * @q - TX queue.
1087 * @tcp_seq
1088 * @tcp_push - tcp push bit.
1089 * @mss - segment size.
1090 * return: NETDEV_TX_BUSY/NET_TX_OK.
1091 */
chcr_ktls_xmit_wr_complete(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tcp_seq,bool is_last_wr,u32 data_len,u32 skb_offset,u32 nfrags,bool tcp_push,u32 mss)1092 static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb,
1093 struct chcr_ktls_info *tx_info,
1094 struct sge_eth_txq *q, u32 tcp_seq,
1095 bool is_last_wr, u32 data_len,
1096 u32 skb_offset, u32 nfrags,
1097 bool tcp_push, u32 mss)
1098 {
1099 u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start;
1100 struct adapter *adap = tx_info->adap;
1101 int credits, left, last_desc;
1102 struct tx_sw_desc *sgl_sdesc;
1103 struct cpl_tx_data *tx_data;
1104 struct cpl_tx_sec_pdu *cpl;
1105 struct ulptx_idata *idata;
1106 struct ulp_txpkt *ulptx;
1107 struct fw_ulptx_wr *wr;
1108 void *pos;
1109 u64 *end;
1110
1111 /* get the number of flits required */
1112 flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len);
1113 /* number of descriptors */
1114 ndesc = chcr_flits_to_desc(flits);
1115 /* check if enough credits available */
1116 credits = chcr_txq_avail(&q->q) - ndesc;
1117 if (unlikely(credits < 0)) {
1118 chcr_eth_txq_stop(q);
1119 return NETDEV_TX_BUSY;
1120 }
1121
1122 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1123 /* Credits are below the threshold vaues, stop the queue after
1124 * injecting the Work Request for this packet.
1125 */
1126 chcr_eth_txq_stop(q);
1127 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1128 }
1129
1130 last_desc = q->q.pidx + ndesc - 1;
1131 if (last_desc >= q->q.size)
1132 last_desc -= q->q.size;
1133 sgl_sdesc = &q->q.sdesc[last_desc];
1134
1135 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1136 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1137 q->mapping_err++;
1138 return NETDEV_TX_BUSY;
1139 }
1140
1141 if (!is_last_wr)
1142 skb_get(skb);
1143
1144 pos = &q->q.desc[q->q.pidx];
1145 end = (u64 *)pos + flits;
1146 /* FW_ULPTX_WR */
1147 wr = pos;
1148 /* WR will need len16 */
1149 len16 = DIV_ROUND_UP(flits, 2);
1150 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1151 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1152 wr->cookie = 0;
1153 pos += sizeof(*wr);
1154 /* ULP_TXPKT */
1155 ulptx = pos;
1156 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1157 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1158 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1159 ULP_TXPKT_RO_F);
1160 ulptx->len = htonl(len16 - 1);
1161 /* ULPTX_IDATA sub-command */
1162 idata = (struct ulptx_idata *)(ulptx + 1);
1163 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1164 /* idata length will include cpl_tx_sec_pdu + key context size +
1165 * cpl_tx_data header.
1166 */
1167 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1168 sizeof(*tx_data));
1169 /* SEC CPL */
1170 cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1171 cpl->op_ivinsrtofst =
1172 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1173 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1174 CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
1175 CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1));
1176 cpl->pldlen = htonl(data_len);
1177
1178 /* encryption should start after tls header size + iv size */
1179 cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1;
1180
1181 cpl->aadstart_cipherstop_hi =
1182 htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |
1183 CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) |
1184 CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1185
1186 /* authentication will also start after tls header + iv size */
1187 cpl->cipherstop_lo_authinsert =
1188 htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) |
1189 CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) |
1190 CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE));
1191
1192 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1193 cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs);
1194 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen);
1195 cpl->scmd1 = cpu_to_be64(tx_info->record_no);
1196
1197 pos = cpl + 1;
1198 /* check if space left to fill the keys */
1199 left = (void *)q->q.stat - pos;
1200 if (!left) {
1201 left = (void *)end - (void *)q->q.stat;
1202 pos = q->q.desc;
1203 end = pos + left;
1204 }
1205
1206 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1207 tx_info->key_ctx_len);
1208 left = (void *)q->q.stat - pos;
1209
1210 if (!left) {
1211 left = (void *)end - (void *)q->q.stat;
1212 pos = q->q.desc;
1213 end = pos + left;
1214 }
1215 /* CPL_TX_DATA */
1216 tx_data = (void *)pos;
1217 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1218 tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(data_len));
1219
1220 tx_data->rsvd = htonl(tcp_seq);
1221
1222 tx_data->flags = htonl(TX_BYPASS_F);
1223 if (tcp_push)
1224 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1225
1226 /* check left again, it might go beyond queue limit */
1227 pos = tx_data + 1;
1228 left = (void *)q->q.stat - pos;
1229
1230 /* check the position again */
1231 if (!left) {
1232 left = (void *)end - (void *)q->q.stat;
1233 pos = q->q.desc;
1234 end = pos + left;
1235 }
1236
1237 /* send the complete packet except the header */
1238 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1239 skb_offset, data_len);
1240 sgl_sdesc->skb = skb;
1241
1242 chcr_txq_advance(&q->q, ndesc);
1243 cxgb4_ring_tx_db(adap, &q->q, ndesc);
1244 atomic64_inc(&adap->ch_ktls_stats.ktls_tx_send_records);
1245
1246 return 0;
1247 }
1248
1249 /*
1250 * chcr_ktls_xmit_wr_short: This is to send out partial records. If its
1251 * a middle part of a record, fetch the prior data to make it 16 byte aligned
1252 * and then only send it out.
1253 *
1254 * @skb - skb contains partial record..
1255 * @tx_info - driver specific tls info.
1256 * @q - TX queue.
1257 * @tcp_seq
1258 * @tcp_push - tcp push bit.
1259 * @mss - segment size.
1260 * @tls_rec_offset - offset from start of the tls record.
1261 * @perior_data - data before the current segment, required to make this record
1262 * 16 byte aligned.
1263 * @prior_data_len - prior_data length (less than 16)
1264 * return: NETDEV_TX_BUSY/NET_TX_OK.
1265 */
chcr_ktls_xmit_wr_short(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tcp_seq,bool tcp_push,u32 mss,u32 tls_rec_offset,u8 * prior_data,u32 prior_data_len,u32 data_len,u32 skb_offset)1266 static int chcr_ktls_xmit_wr_short(struct sk_buff *skb,
1267 struct chcr_ktls_info *tx_info,
1268 struct sge_eth_txq *q,
1269 u32 tcp_seq, bool tcp_push, u32 mss,
1270 u32 tls_rec_offset, u8 *prior_data,
1271 u32 prior_data_len, u32 data_len,
1272 u32 skb_offset)
1273 {
1274 u32 len16, wr_mid = 0, cipher_start, nfrags;
1275 struct adapter *adap = tx_info->adap;
1276 unsigned int flits = 0, ndesc;
1277 int credits, left, last_desc;
1278 struct tx_sw_desc *sgl_sdesc;
1279 struct cpl_tx_data *tx_data;
1280 struct cpl_tx_sec_pdu *cpl;
1281 struct ulptx_idata *idata;
1282 struct ulp_txpkt *ulptx;
1283 struct fw_ulptx_wr *wr;
1284 __be64 iv_record;
1285 void *pos;
1286 u64 *end;
1287
1288 nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1289 /* get the number of flits required, it's a partial record so 2 flits
1290 * (AES_BLOCK_SIZE) will be added.
1291 */
1292 flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len) + 2;
1293 /* get the correct 8 byte IV of this record */
1294 iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no);
1295 /* If it's a middle record and not 16 byte aligned to run AES CTR, need
1296 * to make it 16 byte aligned. So atleadt 2 extra flits of immediate
1297 * data will be added.
1298 */
1299 if (prior_data_len)
1300 flits += 2;
1301 /* number of descriptors */
1302 ndesc = chcr_flits_to_desc(flits);
1303 /* check if enough credits available */
1304 credits = chcr_txq_avail(&q->q) - ndesc;
1305 if (unlikely(credits < 0)) {
1306 chcr_eth_txq_stop(q);
1307 return NETDEV_TX_BUSY;
1308 }
1309
1310 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1311 chcr_eth_txq_stop(q);
1312 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1313 }
1314
1315 last_desc = q->q.pidx + ndesc - 1;
1316 if (last_desc >= q->q.size)
1317 last_desc -= q->q.size;
1318 sgl_sdesc = &q->q.sdesc[last_desc];
1319
1320 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1321 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1322 q->mapping_err++;
1323 return NETDEV_TX_BUSY;
1324 }
1325
1326 pos = &q->q.desc[q->q.pidx];
1327 end = (u64 *)pos + flits;
1328 /* FW_ULPTX_WR */
1329 wr = pos;
1330 /* WR will need len16 */
1331 len16 = DIV_ROUND_UP(flits, 2);
1332 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1333 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1334 wr->cookie = 0;
1335 pos += sizeof(*wr);
1336 /* ULP_TXPKT */
1337 ulptx = pos;
1338 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1339 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1340 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1341 ULP_TXPKT_RO_F);
1342 ulptx->len = htonl(len16 - 1);
1343 /* ULPTX_IDATA sub-command */
1344 idata = (struct ulptx_idata *)(ulptx + 1);
1345 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1346 /* idata length will include cpl_tx_sec_pdu + key context size +
1347 * cpl_tx_data header.
1348 */
1349 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1350 sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len);
1351 /* SEC CPL */
1352 cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1353 /* cipher start will have tls header + iv size extra if its a header
1354 * part of tls record. else only 16 byte IV will be added.
1355 */
1356 cipher_start =
1357 AES_BLOCK_LEN + 1 +
1358 (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0);
1359
1360 cpl->op_ivinsrtofst =
1361 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1362 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1363 CPL_TX_SEC_PDU_IVINSRTOFST_V(1));
1364 cpl->pldlen = htonl(data_len + AES_BLOCK_LEN + prior_data_len);
1365 cpl->aadstart_cipherstop_hi =
1366 htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1367 cpl->cipherstop_lo_authinsert = 0;
1368 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1369 cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs);
1370 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen);
1371 cpl->scmd1 = 0;
1372
1373 pos = cpl + 1;
1374 /* check if space left to fill the keys */
1375 left = (void *)q->q.stat - pos;
1376 if (!left) {
1377 left = (void *)end - (void *)q->q.stat;
1378 pos = q->q.desc;
1379 end = pos + left;
1380 }
1381
1382 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1383 tx_info->key_ctx_len);
1384 left = (void *)q->q.stat - pos;
1385
1386 if (!left) {
1387 left = (void *)end - (void *)q->q.stat;
1388 pos = q->q.desc;
1389 end = pos + left;
1390 }
1391 /* CPL_TX_DATA */
1392 tx_data = (void *)pos;
1393 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1394 tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1395 TX_LENGTH_V(data_len + prior_data_len));
1396 tx_data->rsvd = htonl(tcp_seq);
1397 tx_data->flags = htonl(TX_BYPASS_F);
1398 if (tcp_push)
1399 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1400
1401 /* check left again, it might go beyond queue limit */
1402 pos = tx_data + 1;
1403 left = (void *)q->q.stat - pos;
1404
1405 /* check the position again */
1406 if (!left) {
1407 left = (void *)end - (void *)q->q.stat;
1408 pos = q->q.desc;
1409 end = pos + left;
1410 }
1411 /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8
1412 * bytes of actual IV and 4 bytes of 16 byte-sequence.
1413 */
1414 memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size);
1415 memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size);
1416 *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) =
1417 htonl(2 + (tls_rec_offset ? ((tls_rec_offset -
1418 (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0));
1419
1420 pos += 16;
1421 /* Prior_data_len will always be less than 16 bytes, fill the
1422 * prio_data_len after AES_CTRL_BLOCK and clear the remaining length
1423 * to 0.
1424 */
1425 if (prior_data_len)
1426 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1427 /* send the complete packet except the header */
1428 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1429 skb_offset, data_len);
1430 sgl_sdesc->skb = skb;
1431
1432 chcr_txq_advance(&q->q, ndesc);
1433 cxgb4_ring_tx_db(adap, &q->q, ndesc);
1434
1435 return 0;
1436 }
1437
1438 /*
1439 * chcr_ktls_tx_plaintxt: This handler will take care of the records which has
1440 * only plain text (only tls header and iv)
1441 * @tx_info - driver specific tls info.
1442 * @skb - skb contains partial record..
1443 * @tcp_seq
1444 * @mss - segment size.
1445 * @tcp_push - tcp push bit.
1446 * @q - TX queue.
1447 * @port_id : port number
1448 * @perior_data - data before the current segment, required to make this record
1449 * 16 byte aligned.
1450 * @prior_data_len - prior_data length (less than 16)
1451 * return: NETDEV_TX_BUSY/NET_TX_OK.
1452 */
chcr_ktls_tx_plaintxt(struct chcr_ktls_info * tx_info,struct sk_buff * skb,u32 tcp_seq,u32 mss,bool tcp_push,struct sge_eth_txq * q,u32 port_id,u8 * prior_data,u32 data_len,u32 skb_offset,u32 prior_data_len)1453 static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info,
1454 struct sk_buff *skb, u32 tcp_seq, u32 mss,
1455 bool tcp_push, struct sge_eth_txq *q,
1456 u32 port_id, u8 *prior_data,
1457 u32 data_len, u32 skb_offset,
1458 u32 prior_data_len)
1459 {
1460 int credits, left, len16, last_desc;
1461 unsigned int flits = 0, ndesc;
1462 struct tx_sw_desc *sgl_sdesc;
1463 struct cpl_tx_data *tx_data;
1464 struct ulptx_idata *idata;
1465 struct ulp_txpkt *ulptx;
1466 struct fw_ulptx_wr *wr;
1467 u32 wr_mid = 0, nfrags;
1468 void *pos;
1469 u64 *end;
1470
1471 flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8);
1472 nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1473 flits += chcr_sgl_len(nfrags);
1474 if (prior_data_len)
1475 flits += 2;
1476
1477 /* WR will need len16 */
1478 len16 = DIV_ROUND_UP(flits, 2);
1479 /* check how many descriptors needed */
1480 ndesc = DIV_ROUND_UP(flits, 8);
1481
1482 credits = chcr_txq_avail(&q->q) - ndesc;
1483 if (unlikely(credits < 0)) {
1484 chcr_eth_txq_stop(q);
1485 return NETDEV_TX_BUSY;
1486 }
1487
1488 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1489 chcr_eth_txq_stop(q);
1490 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1491 }
1492
1493 last_desc = q->q.pidx + ndesc - 1;
1494 if (last_desc >= q->q.size)
1495 last_desc -= q->q.size;
1496 sgl_sdesc = &q->q.sdesc[last_desc];
1497
1498 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1499 sgl_sdesc->addr) < 0)) {
1500 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1501 q->mapping_err++;
1502 return NETDEV_TX_BUSY;
1503 }
1504
1505 pos = &q->q.desc[q->q.pidx];
1506 end = (u64 *)pos + flits;
1507 /* FW_ULPTX_WR */
1508 wr = pos;
1509 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1510 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1511 wr->cookie = 0;
1512 pos += sizeof(*wr);
1513 /* ULP_TXPKT */
1514 ulptx = (struct ulp_txpkt *)(wr + 1);
1515 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1516 ULP_TXPKT_DATAMODIFY_V(0) |
1517 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1518 ULP_TXPKT_DEST_V(0) |
1519 ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1));
1520 ulptx->len = htonl(len16 - 1);
1521 /* ULPTX_IDATA sub-command */
1522 idata = (struct ulptx_idata *)(ulptx + 1);
1523 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1524 idata->len = htonl(sizeof(*tx_data) + prior_data_len);
1525 /* CPL_TX_DATA */
1526 tx_data = (struct cpl_tx_data *)(idata + 1);
1527 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1528 tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1529 TX_LENGTH_V(data_len + prior_data_len));
1530 /* set tcp seq number */
1531 tx_data->rsvd = htonl(tcp_seq);
1532 tx_data->flags = htonl(TX_BYPASS_F);
1533 if (tcp_push)
1534 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1535
1536 pos = tx_data + 1;
1537 /* apart from prior_data_len, we should set remaining part of 16 bytes
1538 * to be zero.
1539 */
1540 if (prior_data_len)
1541 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1542
1543 /* check left again, it might go beyond queue limit */
1544 left = (void *)q->q.stat - pos;
1545
1546 /* check the position again */
1547 if (!left) {
1548 left = (void *)end - (void *)q->q.stat;
1549 pos = q->q.desc;
1550 end = pos + left;
1551 }
1552 /* send the complete packet including the header */
1553 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1554 skb_offset, data_len);
1555 sgl_sdesc->skb = skb;
1556
1557 chcr_txq_advance(&q->q, ndesc);
1558 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1559 return 0;
1560 }
1561
chcr_ktls_tunnel_pkt(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct sge_eth_txq * q)1562 static int chcr_ktls_tunnel_pkt(struct chcr_ktls_info *tx_info,
1563 struct sk_buff *skb,
1564 struct sge_eth_txq *q)
1565 {
1566 u32 ctrl, iplen, maclen, wr_mid = 0, len16;
1567 struct tx_sw_desc *sgl_sdesc;
1568 struct fw_eth_tx_pkt_wr *wr;
1569 struct cpl_tx_pkt_core *cpl;
1570 unsigned int flits, ndesc;
1571 int credits, last_desc;
1572 u64 cntrl1, *end;
1573 void *pos;
1574
1575 ctrl = sizeof(*cpl);
1576 flits = DIV_ROUND_UP(sizeof(*wr) + ctrl, 8);
1577
1578 flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags + 1);
1579 len16 = DIV_ROUND_UP(flits, 2);
1580 /* check how many descriptors needed */
1581 ndesc = DIV_ROUND_UP(flits, 8);
1582
1583 credits = chcr_txq_avail(&q->q) - ndesc;
1584 if (unlikely(credits < 0)) {
1585 chcr_eth_txq_stop(q);
1586 return -ENOMEM;
1587 }
1588
1589 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1590 chcr_eth_txq_stop(q);
1591 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1592 }
1593
1594 last_desc = q->q.pidx + ndesc - 1;
1595 if (last_desc >= q->q.size)
1596 last_desc -= q->q.size;
1597 sgl_sdesc = &q->q.sdesc[last_desc];
1598
1599 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1600 sgl_sdesc->addr) < 0)) {
1601 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1602 q->mapping_err++;
1603 return -ENOMEM;
1604 }
1605
1606 iplen = skb_network_header_len(skb);
1607 maclen = skb_mac_header_len(skb);
1608
1609 pos = &q->q.desc[q->q.pidx];
1610 end = (u64 *)pos + flits;
1611 wr = pos;
1612
1613 /* Firmware work request header */
1614 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1615 FW_WR_IMMDLEN_V(ctrl));
1616
1617 wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1618 wr->r3 = 0;
1619
1620 cpl = (void *)(wr + 1);
1621
1622 /* CPL header */
1623 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) |
1624 TXPKT_INTF_V(tx_info->tx_chan) |
1625 TXPKT_PF_V(tx_info->adap->pf));
1626 cpl->pack = 0;
1627 cntrl1 = TXPKT_CSUM_TYPE_V(tx_info->ip_family == AF_INET ?
1628 TX_CSUM_TCPIP : TX_CSUM_TCPIP6);
1629 cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1630 TXPKT_IPHDR_LEN_V(iplen);
1631 /* checksum offload */
1632 cpl->ctrl1 = cpu_to_be64(cntrl1);
1633 cpl->len = htons(skb->len);
1634
1635 pos = cpl + 1;
1636
1637 cxgb4_write_sgl(skb, &q->q, pos, end, 0, sgl_sdesc->addr);
1638 sgl_sdesc->skb = skb;
1639 chcr_txq_advance(&q->q, ndesc);
1640 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1641 return 0;
1642 }
1643
1644 /*
1645 * chcr_ktls_copy_record_in_skb
1646 * @nskb - new skb where the frags to be added.
1647 * @skb - old skb, to copy socket and destructor details.
1648 * @record - specific record which has complete 16k record in frags.
1649 */
chcr_ktls_copy_record_in_skb(struct sk_buff * nskb,struct sk_buff * skb,struct tls_record_info * record)1650 static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,
1651 struct sk_buff *skb,
1652 struct tls_record_info *record)
1653 {
1654 int i = 0;
1655
1656 for (i = 0; i < record->num_frags; i++) {
1657 skb_shinfo(nskb)->frags[i] = record->frags[i];
1658 /* increase the frag ref count */
1659 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
1660 }
1661
1662 skb_shinfo(nskb)->nr_frags = record->num_frags;
1663 nskb->data_len = record->len;
1664 nskb->len += record->len;
1665 nskb->truesize += record->len;
1666 nskb->sk = skb->sk;
1667 nskb->destructor = skb->destructor;
1668 refcount_add(nskb->truesize, &nskb->sk->sk_wmem_alloc);
1669 }
1670
1671 /*
1672 * chcr_ktls_update_snd_una: Reset the SEND_UNA. It will be done to avoid
1673 * sending the same segment again. It will discard the segment which is before
1674 * the current tx max.
1675 * @tx_info - driver specific tls info.
1676 * @q - TX queue.
1677 * return: NET_TX_OK/NET_XMIT_DROP.
1678 */
chcr_ktls_update_snd_una(struct chcr_ktls_info * tx_info,struct sge_eth_txq * q)1679 static int chcr_ktls_update_snd_una(struct chcr_ktls_info *tx_info,
1680 struct sge_eth_txq *q)
1681 {
1682 struct fw_ulptx_wr *wr;
1683 unsigned int ndesc;
1684 int credits;
1685 void *pos;
1686 u32 len;
1687
1688 len = sizeof(*wr) + roundup(CHCR_SET_TCB_FIELD_LEN, 16);
1689 ndesc = DIV_ROUND_UP(len, 64);
1690
1691 credits = chcr_txq_avail(&q->q) - ndesc;
1692 if (unlikely(credits < 0)) {
1693 chcr_eth_txq_stop(q);
1694 return NETDEV_TX_BUSY;
1695 }
1696
1697 pos = &q->q.desc[q->q.pidx];
1698
1699 wr = pos;
1700 /* ULPTX wr */
1701 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1702 wr->cookie = 0;
1703 /* fill len in wr field */
1704 wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
1705
1706 pos += sizeof(*wr);
1707
1708 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
1709 TCB_SND_UNA_RAW_W,
1710 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
1711 TCB_SND_UNA_RAW_V(0), 0);
1712
1713 chcr_txq_advance(&q->q, ndesc);
1714 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1715
1716 return 0;
1717 }
1718
1719 /*
1720 * chcr_end_part_handler: This handler will handle the record which
1721 * is complete or if record's end part is received. T6 adapter has a issue that
1722 * it can't send out TAG with partial record so if its an end part then we have
1723 * to send TAG as well and for which we need to fetch the complete record and
1724 * send it to crypto module.
1725 * @tx_info - driver specific tls info.
1726 * @skb - skb contains partial record.
1727 * @record - complete record of 16K size.
1728 * @tcp_seq
1729 * @mss - segment size in which TP needs to chop a packet.
1730 * @tcp_push_no_fin - tcp push if fin is not set.
1731 * @q - TX queue.
1732 * @tls_end_offset - offset from end of the record.
1733 * @last wr : check if this is the last part of the skb going out.
1734 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1735 */
chcr_end_part_handler(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct tls_record_info * record,u32 tcp_seq,int mss,bool tcp_push_no_fin,struct sge_eth_txq * q,u32 skb_offset,u32 tls_end_offset,bool last_wr)1736 static int chcr_end_part_handler(struct chcr_ktls_info *tx_info,
1737 struct sk_buff *skb,
1738 struct tls_record_info *record,
1739 u32 tcp_seq, int mss, bool tcp_push_no_fin,
1740 struct sge_eth_txq *q, u32 skb_offset,
1741 u32 tls_end_offset, bool last_wr)
1742 {
1743 struct sk_buff *nskb = NULL;
1744 /* check if it is a complete record */
1745 if (tls_end_offset == record->len) {
1746 nskb = skb;
1747 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_complete_pkts);
1748 } else {
1749 nskb = alloc_skb(0, GFP_ATOMIC);
1750 if (!nskb) {
1751 dev_kfree_skb_any(skb);
1752 return NETDEV_TX_BUSY;
1753 }
1754
1755 /* copy complete record in skb */
1756 chcr_ktls_copy_record_in_skb(nskb, skb, record);
1757 /* packet is being sent from the beginning, update the tcp_seq
1758 * accordingly.
1759 */
1760 tcp_seq = tls_record_start_seq(record);
1761 /* reset skb offset */
1762 skb_offset = 0;
1763
1764 if (last_wr)
1765 dev_kfree_skb_any(skb);
1766
1767 last_wr = true;
1768
1769 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_end_pkts);
1770 }
1771
1772 if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq,
1773 last_wr, record->len, skb_offset,
1774 record->num_frags,
1775 (last_wr && tcp_push_no_fin),
1776 mss)) {
1777 goto out;
1778 }
1779 tx_info->prev_seq = record->end_seq;
1780 return 0;
1781 out:
1782 dev_kfree_skb_any(nskb);
1783 return NETDEV_TX_BUSY;
1784 }
1785
1786 /*
1787 * chcr_short_record_handler: This handler will take care of the records which
1788 * doesn't have end part (1st part or the middle part(/s) of a record). In such
1789 * cases, AES CTR will be used in place of AES GCM to send out partial packet.
1790 * This partial record might be the first part of the record, or the middle
1791 * part. In case of middle record we should fetch the prior data to make it 16
1792 * byte aligned. If it has a partial tls header or iv then get to the start of
1793 * tls header. And if it has partial TAG, then remove the complete TAG and send
1794 * only the payload.
1795 * There is one more possibility that it gets a partial header, send that
1796 * portion as a plaintext.
1797 * @tx_info - driver specific tls info.
1798 * @skb - skb contains partial record..
1799 * @record - complete record of 16K size.
1800 * @tcp_seq
1801 * @mss - segment size in which TP needs to chop a packet.
1802 * @tcp_push_no_fin - tcp push if fin is not set.
1803 * @q - TX queue.
1804 * @tls_end_offset - offset from end of the record.
1805 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1806 */
chcr_short_record_handler(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct tls_record_info * record,u32 tcp_seq,int mss,bool tcp_push_no_fin,u32 data_len,u32 skb_offset,struct sge_eth_txq * q,u32 tls_end_offset)1807 static int chcr_short_record_handler(struct chcr_ktls_info *tx_info,
1808 struct sk_buff *skb,
1809 struct tls_record_info *record,
1810 u32 tcp_seq, int mss, bool tcp_push_no_fin,
1811 u32 data_len, u32 skb_offset,
1812 struct sge_eth_txq *q, u32 tls_end_offset)
1813 {
1814 u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record);
1815 u8 prior_data[16] = {0};
1816 u32 prior_data_len = 0;
1817
1818 /* check if the skb is ending in middle of tag/HASH, its a big
1819 * trouble, send the packet before the HASH.
1820 */
1821 int remaining_record = tls_end_offset - data_len;
1822
1823 if (remaining_record > 0 &&
1824 remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) {
1825 int trimmed_len = 0;
1826
1827 if (tls_end_offset > TLS_CIPHER_AES_GCM_128_TAG_SIZE)
1828 trimmed_len = data_len -
1829 (TLS_CIPHER_AES_GCM_128_TAG_SIZE -
1830 remaining_record);
1831 if (!trimmed_len)
1832 return FALLBACK;
1833
1834 WARN_ON(trimmed_len > data_len);
1835
1836 data_len = trimmed_len;
1837 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_trimmed_pkts);
1838 }
1839
1840 /* check if it is only the header part. */
1841 if (tls_rec_offset + data_len <= (TLS_HEADER_SIZE + tx_info->iv_size)) {
1842 if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
1843 tcp_push_no_fin, q,
1844 tx_info->port_id, prior_data,
1845 data_len, skb_offset, prior_data_len))
1846 goto out;
1847
1848 tx_info->prev_seq = tcp_seq + data_len;
1849 return 0;
1850 }
1851
1852 /* check if the middle record's start point is 16 byte aligned. CTR
1853 * needs 16 byte aligned start point to start encryption.
1854 */
1855 if (tls_rec_offset) {
1856 /* there is an offset from start, means its a middle record */
1857 int remaining = 0;
1858
1859 if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) {
1860 prior_data_len = tls_rec_offset;
1861 tls_rec_offset = 0;
1862 remaining = 0;
1863 } else {
1864 prior_data_len =
1865 (tls_rec_offset -
1866 (TLS_HEADER_SIZE + tx_info->iv_size))
1867 % AES_BLOCK_LEN;
1868 remaining = tls_rec_offset - prior_data_len;
1869 }
1870
1871 /* if prior_data_len is not zero, means we need to fetch prior
1872 * data to make this record 16 byte aligned, or we need to reach
1873 * to start offset.
1874 */
1875 if (prior_data_len) {
1876 int i = 0;
1877 u8 *data = NULL;
1878 skb_frag_t *f;
1879 u8 *vaddr;
1880 int frag_size = 0, frag_delta = 0;
1881
1882 while (remaining > 0) {
1883 frag_size = skb_frag_size(&record->frags[i]);
1884 if (remaining < frag_size)
1885 break;
1886
1887 remaining -= frag_size;
1888 i++;
1889 }
1890 f = &record->frags[i];
1891 vaddr = kmap_atomic(skb_frag_page(f));
1892
1893 data = vaddr + skb_frag_off(f) + remaining;
1894 frag_delta = skb_frag_size(f) - remaining;
1895
1896 if (frag_delta >= prior_data_len) {
1897 memcpy(prior_data, data, prior_data_len);
1898 kunmap_atomic(vaddr);
1899 } else {
1900 memcpy(prior_data, data, frag_delta);
1901 kunmap_atomic(vaddr);
1902 /* get the next page */
1903 f = &record->frags[i + 1];
1904 vaddr = kmap_atomic(skb_frag_page(f));
1905 data = vaddr + skb_frag_off(f);
1906 memcpy(prior_data + frag_delta,
1907 data, (prior_data_len - frag_delta));
1908 kunmap_atomic(vaddr);
1909 }
1910 /* reset tcp_seq as per the prior_data_required len */
1911 tcp_seq -= prior_data_len;
1912 }
1913 /* reset snd una, so the middle record won't send the already
1914 * sent part.
1915 */
1916 if (chcr_ktls_update_snd_una(tx_info, q))
1917 goto out;
1918 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_middle_pkts);
1919 } else {
1920 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_start_pkts);
1921 }
1922
1923 if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin,
1924 mss, tls_rec_offset, prior_data,
1925 prior_data_len, data_len, skb_offset)) {
1926 goto out;
1927 }
1928
1929 tx_info->prev_seq = tcp_seq + data_len + prior_data_len;
1930 return 0;
1931 out:
1932 dev_kfree_skb_any(skb);
1933 return NETDEV_TX_BUSY;
1934 }
1935
chcr_ktls_sw_fallback(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q)1936 static int chcr_ktls_sw_fallback(struct sk_buff *skb,
1937 struct chcr_ktls_info *tx_info,
1938 struct sge_eth_txq *q)
1939 {
1940 u32 data_len, skb_offset;
1941 struct sk_buff *nskb;
1942 struct tcphdr *th;
1943
1944 nskb = tls_encrypt_skb(skb);
1945
1946 if (!nskb)
1947 return 0;
1948
1949 th = tcp_hdr(nskb);
1950 skb_offset = skb_transport_offset(nskb) + tcp_hdrlen(nskb);
1951 data_len = nskb->len - skb_offset;
1952 skb_tx_timestamp(nskb);
1953
1954 if (chcr_ktls_tunnel_pkt(tx_info, nskb, q))
1955 goto out;
1956
1957 tx_info->prev_seq = ntohl(th->seq) + data_len;
1958 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_fallback);
1959 return 0;
1960 out:
1961 dev_kfree_skb_any(nskb);
1962 return 0;
1963 }
1964 /* nic tls TX handler */
chcr_ktls_xmit(struct sk_buff * skb,struct net_device * dev)1965 static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev)
1966 {
1967 u32 tls_end_offset, tcp_seq, skb_data_len, skb_offset;
1968 struct ch_ktls_port_stats_debug *port_stats;
1969 struct chcr_ktls_ofld_ctx_tx *tx_ctx;
1970 struct ch_ktls_stats_debug *stats;
1971 struct tcphdr *th = tcp_hdr(skb);
1972 int data_len, qidx, ret = 0, mss;
1973 struct tls_record_info *record;
1974 struct chcr_ktls_info *tx_info;
1975 struct tls_context *tls_ctx;
1976 struct sge_eth_txq *q;
1977 struct adapter *adap;
1978 unsigned long flags;
1979
1980 tcp_seq = ntohl(th->seq);
1981 skb_offset = skb_transport_offset(skb) + tcp_hdrlen(skb);
1982 skb_data_len = skb->len - skb_offset;
1983 data_len = skb_data_len;
1984
1985 mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : data_len;
1986
1987 tls_ctx = tls_get_ctx(skb->sk);
1988 if (unlikely(tls_ctx->netdev != dev))
1989 goto out;
1990
1991 tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
1992 tx_info = tx_ctx->chcr_info;
1993
1994 if (unlikely(!tx_info))
1995 goto out;
1996
1997 adap = tx_info->adap;
1998 stats = &adap->ch_ktls_stats;
1999 port_stats = &stats->ktls_port[tx_info->port_id];
2000
2001 qidx = skb->queue_mapping;
2002 q = &adap->sge.ethtxq[qidx + tx_info->first_qset];
2003 cxgb4_reclaim_completed_tx(adap, &q->q, true);
2004 /* if tcp options are set but finish is not send the options first */
2005 if (!th->fin && chcr_ktls_check_tcp_options(th)) {
2006 ret = chcr_ktls_write_tcp_options(tx_info, skb, q,
2007 tx_info->tx_chan);
2008 if (ret)
2009 return NETDEV_TX_BUSY;
2010 }
2011
2012 /* TCP segments can be in received either complete or partial.
2013 * chcr_end_part_handler will handle cases if complete record or end
2014 * part of the record is received. Incase of partial end part of record,
2015 * we will send the complete record again.
2016 */
2017
2018 do {
2019 int i;
2020
2021 cxgb4_reclaim_completed_tx(adap, &q->q, true);
2022 /* lock taken */
2023 spin_lock_irqsave(&tx_ctx->base.lock, flags);
2024 /* fetch the tls record */
2025 record = tls_get_record(&tx_ctx->base, tcp_seq,
2026 &tx_info->record_no);
2027 /* By the time packet reached to us, ACK is received, and record
2028 * won't be found in that case, handle it gracefully.
2029 */
2030 if (unlikely(!record)) {
2031 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2032 atomic64_inc(&port_stats->ktls_tx_drop_no_sync_data);
2033 goto out;
2034 }
2035
2036 tls_end_offset = record->end_seq - tcp_seq;
2037
2038 pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",
2039 tcp_seq, record->end_seq, tx_info->prev_seq, data_len);
2040 /* update tcb for the skb */
2041 if (skb_data_len == data_len) {
2042 u32 tx_max = tcp_seq;
2043
2044 if (!tls_record_is_start_marker(record) &&
2045 tls_end_offset < TLS_CIPHER_AES_GCM_128_TAG_SIZE)
2046 tx_max = record->end_seq -
2047 TLS_CIPHER_AES_GCM_128_TAG_SIZE;
2048
2049 ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, tx_max,
2050 ntohl(th->ack_seq),
2051 ntohs(th->window),
2052 tls_end_offset !=
2053 record->len);
2054 if (ret) {
2055 spin_unlock_irqrestore(&tx_ctx->base.lock,
2056 flags);
2057 goto out;
2058 }
2059
2060 if (th->fin)
2061 skb_get(skb);
2062 }
2063
2064 if (unlikely(tls_record_is_start_marker(record))) {
2065 atomic64_inc(&port_stats->ktls_tx_skip_no_sync_data);
2066 /* If tls_end_offset < data_len, means there is some
2067 * data after start marker, which needs encryption, send
2068 * plaintext first and take skb refcount. else send out
2069 * complete pkt as plaintext.
2070 */
2071 if (tls_end_offset < data_len)
2072 skb_get(skb);
2073 else
2074 tls_end_offset = data_len;
2075
2076 ret = chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
2077 (!th->fin && th->psh), q,
2078 tx_info->port_id, NULL,
2079 tls_end_offset, skb_offset,
2080 0);
2081
2082 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2083 if (ret) {
2084 /* free the refcount taken earlier */
2085 if (tls_end_offset < data_len)
2086 dev_kfree_skb_any(skb);
2087 goto out;
2088 }
2089
2090 data_len -= tls_end_offset;
2091 tcp_seq = record->end_seq;
2092 skb_offset += tls_end_offset;
2093 continue;
2094 }
2095
2096 /* increase page reference count of the record, so that there
2097 * won't be any chance of page free in middle if in case stack
2098 * receives ACK and try to delete the record.
2099 */
2100 for (i = 0; i < record->num_frags; i++)
2101 __skb_frag_ref(&record->frags[i]);
2102 /* lock cleared */
2103 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2104
2105
2106 /* if a tls record is finishing in this SKB */
2107 if (tls_end_offset <= data_len) {
2108 ret = chcr_end_part_handler(tx_info, skb, record,
2109 tcp_seq, mss,
2110 (!th->fin && th->psh), q,
2111 skb_offset,
2112 tls_end_offset,
2113 skb_offset +
2114 tls_end_offset == skb->len);
2115
2116 data_len -= tls_end_offset;
2117 /* tcp_seq increment is required to handle next record.
2118 */
2119 tcp_seq += tls_end_offset;
2120 skb_offset += tls_end_offset;
2121 } else {
2122 ret = chcr_short_record_handler(tx_info, skb,
2123 record, tcp_seq, mss,
2124 (!th->fin && th->psh),
2125 data_len, skb_offset,
2126 q, tls_end_offset);
2127 data_len = 0;
2128 }
2129
2130 /* clear the frag ref count which increased locally before */
2131 for (i = 0; i < record->num_frags; i++) {
2132 /* clear the frag ref count */
2133 __skb_frag_unref(&record->frags[i]);
2134 }
2135 /* if any failure, come out from the loop. */
2136 if (ret) {
2137 if (th->fin)
2138 dev_kfree_skb_any(skb);
2139
2140 if (ret == FALLBACK)
2141 return chcr_ktls_sw_fallback(skb, tx_info, q);
2142
2143 return NETDEV_TX_OK;
2144 }
2145
2146 /* length should never be less than 0 */
2147 WARN_ON(data_len < 0);
2148
2149 } while (data_len > 0);
2150
2151 atomic64_inc(&port_stats->ktls_tx_encrypted_packets);
2152 atomic64_add(skb_data_len, &port_stats->ktls_tx_encrypted_bytes);
2153
2154 /* tcp finish is set, send a separate tcp msg including all the options
2155 * as well.
2156 */
2157 if (th->fin) {
2158 chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan);
2159 dev_kfree_skb_any(skb);
2160 }
2161
2162 return NETDEV_TX_OK;
2163 out:
2164 dev_kfree_skb_any(skb);
2165 return NETDEV_TX_OK;
2166 }
2167
chcr_ktls_uld_add(const struct cxgb4_lld_info * lldi)2168 static void *chcr_ktls_uld_add(const struct cxgb4_lld_info *lldi)
2169 {
2170 struct chcr_ktls_uld_ctx *u_ctx;
2171
2172 pr_info_once("%s - version %s\n", CHCR_KTLS_DRV_DESC,
2173 CHCR_KTLS_DRV_VERSION);
2174 u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
2175 if (!u_ctx) {
2176 u_ctx = ERR_PTR(-ENOMEM);
2177 goto out;
2178 }
2179 u_ctx->lldi = *lldi;
2180 out:
2181 return u_ctx;
2182 }
2183
2184 static const struct tlsdev_ops chcr_ktls_ops = {
2185 .tls_dev_add = chcr_ktls_dev_add,
2186 .tls_dev_del = chcr_ktls_dev_del,
2187 };
2188
2189 static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
2190 [CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl,
2191 [CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl,
2192 };
2193
chcr_ktls_uld_rx_handler(void * handle,const __be64 * rsp,const struct pkt_gl * pgl)2194 static int chcr_ktls_uld_rx_handler(void *handle, const __be64 *rsp,
2195 const struct pkt_gl *pgl)
2196 {
2197 const struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)rsp;
2198 struct chcr_ktls_uld_ctx *u_ctx = handle;
2199 u8 opcode = rpl->ot.opcode;
2200 struct adapter *adap;
2201
2202 adap = pci_get_drvdata(u_ctx->lldi.pdev);
2203
2204 if (!work_handlers[opcode]) {
2205 pr_err("Unsupported opcode %d received\n", opcode);
2206 return 0;
2207 }
2208
2209 work_handlers[opcode](adap, (unsigned char *)&rsp[1]);
2210 return 0;
2211 }
2212
chcr_ktls_uld_state_change(void * handle,enum cxgb4_state new_state)2213 static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state)
2214 {
2215 struct chcr_ktls_uld_ctx *u_ctx = handle;
2216
2217 switch (new_state) {
2218 case CXGB4_STATE_UP:
2219 pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev));
2220 mutex_lock(&dev_mutex);
2221 list_add_tail(&u_ctx->entry, &uld_ctx_list);
2222 mutex_unlock(&dev_mutex);
2223 break;
2224 case CXGB4_STATE_START_RECOVERY:
2225 case CXGB4_STATE_DOWN:
2226 case CXGB4_STATE_DETACH:
2227 pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev));
2228 mutex_lock(&dev_mutex);
2229 list_del(&u_ctx->entry);
2230 mutex_unlock(&dev_mutex);
2231 break;
2232 default:
2233 break;
2234 }
2235
2236 return 0;
2237 }
2238
2239 static struct cxgb4_uld_info chcr_ktls_uld_info = {
2240 .name = CHCR_KTLS_DRV_MODULE_NAME,
2241 .nrxq = 1,
2242 .rxq_size = 1024,
2243 .add = chcr_ktls_uld_add,
2244 .tx_handler = chcr_ktls_xmit,
2245 .rx_handler = chcr_ktls_uld_rx_handler,
2246 .state_change = chcr_ktls_uld_state_change,
2247 .tlsdev_ops = &chcr_ktls_ops,
2248 };
2249
chcr_ktls_init(void)2250 static int __init chcr_ktls_init(void)
2251 {
2252 cxgb4_register_uld(CXGB4_ULD_KTLS, &chcr_ktls_uld_info);
2253 return 0;
2254 }
2255
chcr_ktls_exit(void)2256 static void __exit chcr_ktls_exit(void)
2257 {
2258 struct chcr_ktls_uld_ctx *u_ctx, *tmp;
2259 struct adapter *adap;
2260
2261 pr_info("driver unloaded\n");
2262
2263 mutex_lock(&dev_mutex);
2264 list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
2265 adap = pci_get_drvdata(u_ctx->lldi.pdev);
2266 memset(&adap->ch_ktls_stats, 0, sizeof(adap->ch_ktls_stats));
2267 list_del(&u_ctx->entry);
2268 kfree(u_ctx);
2269 }
2270 mutex_unlock(&dev_mutex);
2271 cxgb4_unregister_uld(CXGB4_ULD_KTLS);
2272 }
2273
2274 module_init(chcr_ktls_init);
2275 module_exit(chcr_ktls_exit);
2276
2277 MODULE_DESCRIPTION("Chelsio NIC TLS ULD driver");
2278 MODULE_LICENSE("GPL");
2279 MODULE_AUTHOR("Chelsio Communications");
2280 MODULE_VERSION(CHCR_KTLS_DRV_VERSION);
2281