1 /* 2 * QEMU TX packets abstractions 3 * 4 * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com) 5 * 6 * Developed by Daynix Computing LTD (http://www.daynix.com) 7 * 8 * Authors: 9 * Dmitry Fleytman <dmitry@daynix.com> 10 * Tamir Shomer <tamirs@daynix.com> 11 * Yan Vugenfirer <yan@daynix.com> 12 * 13 * This work is licensed under the terms of the GNU GPL, version 2 or later. 14 * See the COPYING file in the top-level directory. 15 * 16 */ 17 18 #include "qemu/osdep.h" 19 #include "net_tx_pkt.h" 20 #include "net/eth.h" 21 #include "net/checksum.h" 22 #include "net/tap.h" 23 #include "net/net.h" 24 #include "hw/pci/pci_device.h" 25 26 enum { 27 NET_TX_PKT_VHDR_FRAG = 0, 28 NET_TX_PKT_L2HDR_FRAG, 29 NET_TX_PKT_L3HDR_FRAG, 30 NET_TX_PKT_PL_START_FRAG 31 }; 32 33 /* TX packet private context */ 34 struct NetTxPkt { 35 PCIDevice *pci_dev; 36 37 struct virtio_net_hdr virt_hdr; 38 39 struct iovec *raw; 40 uint32_t raw_frags; 41 uint32_t max_raw_frags; 42 43 struct iovec *vec; 44 45 uint8_t l2_hdr[ETH_MAX_L2_HDR_LEN]; 46 union { 47 struct ip_header ip; 48 struct ip6_header ip6; 49 uint8_t octets[ETH_MAX_IP_DGRAM_LEN]; 50 } l3_hdr; 51 52 uint32_t payload_len; 53 54 uint32_t payload_frags; 55 uint32_t max_payload_frags; 56 57 uint16_t hdr_len; 58 eth_pkt_types_e packet_type; 59 uint8_t l4proto; 60 }; 61 62 void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev, 63 uint32_t max_frags) 64 { 65 struct NetTxPkt *p = g_malloc0(sizeof *p); 66 67 p->pci_dev = pci_dev; 68 69 p->vec = g_new(struct iovec, max_frags + NET_TX_PKT_PL_START_FRAG); 70 71 p->raw = g_new(struct iovec, max_frags); 72 73 p->max_payload_frags = max_frags; 74 p->max_raw_frags = max_frags; 75 p->vec[NET_TX_PKT_VHDR_FRAG].iov_base = &p->virt_hdr; 76 p->vec[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof p->virt_hdr; 77 p->vec[NET_TX_PKT_L2HDR_FRAG].iov_base = &p->l2_hdr; 78 p->vec[NET_TX_PKT_L3HDR_FRAG].iov_base = &p->l3_hdr; 79 80 *pkt = p; 81 } 82 83 void net_tx_pkt_uninit(struct NetTxPkt *pkt) 84 { 85 if (pkt) { 86 g_free(pkt->vec); 87 g_free(pkt->raw); 88 g_free(pkt); 89 } 90 } 91 92 void net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt *pkt) 93 { 94 uint16_t csum; 95 assert(pkt); 96 97 pkt->l3_hdr.ip.ip_len = cpu_to_be16(pkt->payload_len + 98 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len); 99 100 pkt->l3_hdr.ip.ip_sum = 0; 101 csum = net_raw_checksum(pkt->l3_hdr.octets, 102 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len); 103 pkt->l3_hdr.ip.ip_sum = cpu_to_be16(csum); 104 } 105 106 void net_tx_pkt_update_ip_checksums(struct NetTxPkt *pkt) 107 { 108 uint16_t csum; 109 uint32_t cntr, cso; 110 assert(pkt); 111 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; 112 void *ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base; 113 114 if (pkt->payload_len + pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len > 115 ETH_MAX_IP_DGRAM_LEN) { 116 return; 117 } 118 119 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 120 gso_type == VIRTIO_NET_HDR_GSO_UDP) { 121 /* Calculate IP header checksum */ 122 net_tx_pkt_update_ip_hdr_checksum(pkt); 123 124 /* Calculate IP pseudo header checksum */ 125 cntr = eth_calc_ip4_pseudo_hdr_csum(ip_hdr, pkt->payload_len, &cso); 126 csum = cpu_to_be16(~net_checksum_finish(cntr)); 127 } else if (gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 128 /* Calculate IP pseudo header checksum */ 129 cntr = eth_calc_ip6_pseudo_hdr_csum(ip_hdr, pkt->payload_len, 130 IP_PROTO_TCP, &cso); 131 csum = cpu_to_be16(~net_checksum_finish(cntr)); 132 } else { 133 return; 134 } 135 136 iov_from_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags, 137 pkt->virt_hdr.csum_offset, &csum, sizeof(csum)); 138 } 139 140 static void net_tx_pkt_calculate_hdr_len(struct NetTxPkt *pkt) 141 { 142 pkt->hdr_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len + 143 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len; 144 } 145 146 static bool net_tx_pkt_parse_headers(struct NetTxPkt *pkt) 147 { 148 struct iovec *l2_hdr, *l3_hdr; 149 size_t bytes_read; 150 size_t full_ip6hdr_len; 151 uint16_t l3_proto; 152 153 assert(pkt); 154 155 l2_hdr = &pkt->vec[NET_TX_PKT_L2HDR_FRAG]; 156 l3_hdr = &pkt->vec[NET_TX_PKT_L3HDR_FRAG]; 157 158 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 0, l2_hdr->iov_base, 159 ETH_MAX_L2_HDR_LEN); 160 if (bytes_read < sizeof(struct eth_header)) { 161 l2_hdr->iov_len = 0; 162 return false; 163 } 164 165 l2_hdr->iov_len = sizeof(struct eth_header); 166 switch (be16_to_cpu(PKT_GET_ETH_HDR(l2_hdr->iov_base)->h_proto)) { 167 case ETH_P_VLAN: 168 l2_hdr->iov_len += sizeof(struct vlan_header); 169 break; 170 case ETH_P_DVLAN: 171 l2_hdr->iov_len += 2 * sizeof(struct vlan_header); 172 break; 173 } 174 175 if (bytes_read < l2_hdr->iov_len) { 176 l2_hdr->iov_len = 0; 177 l3_hdr->iov_len = 0; 178 pkt->packet_type = ETH_PKT_UCAST; 179 return false; 180 } else { 181 l2_hdr->iov_len = ETH_MAX_L2_HDR_LEN; 182 l2_hdr->iov_len = eth_get_l2_hdr_length(l2_hdr->iov_base); 183 pkt->packet_type = get_eth_packet_type(l2_hdr->iov_base); 184 } 185 186 l3_proto = eth_get_l3_proto(l2_hdr, 1, l2_hdr->iov_len); 187 188 switch (l3_proto) { 189 case ETH_P_IP: 190 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len, 191 l3_hdr->iov_base, sizeof(struct ip_header)); 192 193 if (bytes_read < sizeof(struct ip_header)) { 194 l3_hdr->iov_len = 0; 195 return false; 196 } 197 198 l3_hdr->iov_len = IP_HDR_GET_LEN(l3_hdr->iov_base); 199 200 if (l3_hdr->iov_len < sizeof(struct ip_header)) { 201 l3_hdr->iov_len = 0; 202 return false; 203 } 204 205 pkt->l4proto = IP_HDR_GET_P(l3_hdr->iov_base); 206 207 if (IP_HDR_GET_LEN(l3_hdr->iov_base) != sizeof(struct ip_header)) { 208 /* copy optional IPv4 header data if any*/ 209 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 210 l2_hdr->iov_len + sizeof(struct ip_header), 211 l3_hdr->iov_base + sizeof(struct ip_header), 212 l3_hdr->iov_len - sizeof(struct ip_header)); 213 if (bytes_read < l3_hdr->iov_len - sizeof(struct ip_header)) { 214 l3_hdr->iov_len = 0; 215 return false; 216 } 217 } 218 219 break; 220 221 case ETH_P_IPV6: 222 { 223 eth_ip6_hdr_info hdrinfo; 224 225 if (!eth_parse_ipv6_hdr(pkt->raw, pkt->raw_frags, l2_hdr->iov_len, 226 &hdrinfo)) { 227 l3_hdr->iov_len = 0; 228 return false; 229 } 230 231 pkt->l4proto = hdrinfo.l4proto; 232 full_ip6hdr_len = hdrinfo.full_hdr_len; 233 234 if (full_ip6hdr_len > ETH_MAX_IP_DGRAM_LEN) { 235 l3_hdr->iov_len = 0; 236 return false; 237 } 238 239 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len, 240 l3_hdr->iov_base, full_ip6hdr_len); 241 242 if (bytes_read < full_ip6hdr_len) { 243 l3_hdr->iov_len = 0; 244 return false; 245 } else { 246 l3_hdr->iov_len = full_ip6hdr_len; 247 } 248 break; 249 } 250 default: 251 l3_hdr->iov_len = 0; 252 break; 253 } 254 255 net_tx_pkt_calculate_hdr_len(pkt); 256 return true; 257 } 258 259 static void net_tx_pkt_rebuild_payload(struct NetTxPkt *pkt) 260 { 261 pkt->payload_len = iov_size(pkt->raw, pkt->raw_frags) - pkt->hdr_len; 262 pkt->payload_frags = iov_copy(&pkt->vec[NET_TX_PKT_PL_START_FRAG], 263 pkt->max_payload_frags, 264 pkt->raw, pkt->raw_frags, 265 pkt->hdr_len, pkt->payload_len); 266 } 267 268 bool net_tx_pkt_parse(struct NetTxPkt *pkt) 269 { 270 if (net_tx_pkt_parse_headers(pkt)) { 271 net_tx_pkt_rebuild_payload(pkt); 272 return true; 273 } else { 274 return false; 275 } 276 } 277 278 struct virtio_net_hdr *net_tx_pkt_get_vhdr(struct NetTxPkt *pkt) 279 { 280 assert(pkt); 281 return &pkt->virt_hdr; 282 } 283 284 static uint8_t net_tx_pkt_get_gso_type(struct NetTxPkt *pkt, 285 bool tso_enable) 286 { 287 uint8_t rc = VIRTIO_NET_HDR_GSO_NONE; 288 uint16_t l3_proto; 289 290 l3_proto = eth_get_l3_proto(&pkt->vec[NET_TX_PKT_L2HDR_FRAG], 1, 291 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len); 292 293 if (!tso_enable) { 294 goto func_exit; 295 } 296 297 rc = eth_get_gso_type(l3_proto, pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base, 298 pkt->l4proto); 299 300 func_exit: 301 return rc; 302 } 303 304 bool net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable, 305 bool csum_enable, uint32_t gso_size) 306 { 307 struct tcp_hdr l4hdr; 308 size_t bytes_read; 309 assert(pkt); 310 311 /* csum has to be enabled if tso is. */ 312 assert(csum_enable || !tso_enable); 313 314 pkt->virt_hdr.gso_type = net_tx_pkt_get_gso_type(pkt, tso_enable); 315 316 switch (pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 317 case VIRTIO_NET_HDR_GSO_NONE: 318 pkt->virt_hdr.hdr_len = 0; 319 pkt->virt_hdr.gso_size = 0; 320 break; 321 322 case VIRTIO_NET_HDR_GSO_UDP: 323 pkt->virt_hdr.gso_size = gso_size; 324 pkt->virt_hdr.hdr_len = pkt->hdr_len + sizeof(struct udp_header); 325 break; 326 327 case VIRTIO_NET_HDR_GSO_TCPV4: 328 case VIRTIO_NET_HDR_GSO_TCPV6: 329 bytes_read = iov_to_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], 330 pkt->payload_frags, 0, &l4hdr, sizeof(l4hdr)); 331 if (bytes_read < sizeof(l4hdr) || 332 l4hdr.th_off * sizeof(uint32_t) < sizeof(l4hdr)) { 333 return false; 334 } 335 336 pkt->virt_hdr.hdr_len = pkt->hdr_len + l4hdr.th_off * sizeof(uint32_t); 337 pkt->virt_hdr.gso_size = gso_size; 338 break; 339 340 default: 341 g_assert_not_reached(); 342 } 343 344 if (csum_enable) { 345 switch (pkt->l4proto) { 346 case IP_PROTO_TCP: 347 if (pkt->payload_len < sizeof(struct tcp_hdr)) { 348 return false; 349 } 350 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 351 pkt->virt_hdr.csum_start = pkt->hdr_len; 352 pkt->virt_hdr.csum_offset = offsetof(struct tcp_hdr, th_sum); 353 break; 354 case IP_PROTO_UDP: 355 if (pkt->payload_len < sizeof(struct udp_hdr)) { 356 return false; 357 } 358 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 359 pkt->virt_hdr.csum_start = pkt->hdr_len; 360 pkt->virt_hdr.csum_offset = offsetof(struct udp_hdr, uh_sum); 361 break; 362 default: 363 break; 364 } 365 } 366 367 return true; 368 } 369 370 void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt, 371 uint16_t vlan, uint16_t vlan_ethtype) 372 { 373 bool is_new; 374 assert(pkt); 375 376 eth_setup_vlan_headers_ex(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base, 377 vlan, vlan_ethtype, &is_new); 378 379 /* update l2hdrlen */ 380 if (is_new) { 381 pkt->hdr_len += sizeof(struct vlan_header); 382 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len += 383 sizeof(struct vlan_header); 384 } 385 } 386 387 static bool net_tx_pkt_add_raw_fragment_common(struct NetTxPkt *pkt, 388 void *base, size_t len) 389 { 390 struct iovec *ventry; 391 assert(pkt); 392 393 if (pkt->raw_frags >= pkt->max_raw_frags) { 394 return false; 395 } 396 397 ventry = &pkt->raw[pkt->raw_frags]; 398 ventry->iov_base = base; 399 ventry->iov_len = len; 400 pkt->raw_frags++; 401 402 return true; 403 } 404 405 bool net_tx_pkt_has_fragments(struct NetTxPkt *pkt) 406 { 407 return pkt->raw_frags > 0; 408 } 409 410 eth_pkt_types_e net_tx_pkt_get_packet_type(struct NetTxPkt *pkt) 411 { 412 assert(pkt); 413 414 return pkt->packet_type; 415 } 416 417 size_t net_tx_pkt_get_total_len(struct NetTxPkt *pkt) 418 { 419 assert(pkt); 420 421 return pkt->hdr_len + pkt->payload_len; 422 } 423 424 void net_tx_pkt_dump(struct NetTxPkt *pkt) 425 { 426 #ifdef NET_TX_PKT_DEBUG 427 assert(pkt); 428 429 printf("TX PKT: hdr_len: %d, pkt_type: 0x%X, l2hdr_len: %lu, " 430 "l3hdr_len: %lu, payload_len: %u\n", pkt->hdr_len, pkt->packet_type, 431 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len, 432 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len, pkt->payload_len); 433 #endif 434 } 435 436 void net_tx_pkt_reset(struct NetTxPkt *pkt, PCIDevice *pci_dev) 437 { 438 int i; 439 440 /* no assert, as reset can be called before tx_pkt_init */ 441 if (!pkt) { 442 return; 443 } 444 445 memset(&pkt->virt_hdr, 0, sizeof(pkt->virt_hdr)); 446 447 assert(pkt->vec); 448 449 pkt->payload_len = 0; 450 pkt->payload_frags = 0; 451 452 if (pkt->max_raw_frags > 0) { 453 assert(pkt->raw); 454 for (i = 0; i < pkt->raw_frags; i++) { 455 assert(pkt->raw[i].iov_base); 456 net_tx_pkt_unmap_frag_pci(pkt->pci_dev, 457 pkt->raw[i].iov_base, 458 pkt->raw[i].iov_len); 459 } 460 } 461 pkt->pci_dev = pci_dev; 462 pkt->raw_frags = 0; 463 464 pkt->hdr_len = 0; 465 pkt->l4proto = 0; 466 } 467 468 void net_tx_pkt_unmap_frag_pci(void *context, void *base, size_t len) 469 { 470 pci_dma_unmap(context, base, len, DMA_DIRECTION_TO_DEVICE, 0); 471 } 472 473 bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, hwaddr pa, 474 size_t len) 475 { 476 dma_addr_t mapped_len = len; 477 void *base = pci_dma_map(pkt->pci_dev, pa, &mapped_len, 478 DMA_DIRECTION_TO_DEVICE); 479 if (!base) { 480 return false; 481 } 482 483 if (mapped_len != len || 484 !net_tx_pkt_add_raw_fragment_common(pkt, base, len)) { 485 net_tx_pkt_unmap_frag_pci(pkt->pci_dev, base, mapped_len); 486 return false; 487 } 488 489 return true; 490 } 491 492 static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt, 493 struct iovec *iov, uint32_t iov_len, 494 uint16_t csl) 495 { 496 uint32_t csum_cntr; 497 uint16_t csum = 0; 498 uint32_t cso; 499 /* num of iovec without vhdr */ 500 size_t csum_offset = pkt->virt_hdr.csum_start + pkt->virt_hdr.csum_offset; 501 uint16_t l3_proto = eth_get_l3_proto(iov, 1, iov->iov_len); 502 503 /* Put zero to checksum field */ 504 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum); 505 506 /* Calculate L4 TCP/UDP checksum */ 507 csum_cntr = 0; 508 cso = 0; 509 /* add pseudo header to csum */ 510 if (l3_proto == ETH_P_IP) { 511 csum_cntr = eth_calc_ip4_pseudo_hdr_csum( 512 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base, 513 csl, &cso); 514 } else if (l3_proto == ETH_P_IPV6) { 515 csum_cntr = eth_calc_ip6_pseudo_hdr_csum( 516 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base, 517 csl, pkt->l4proto, &cso); 518 } 519 520 /* data checksum */ 521 csum_cntr += 522 net_checksum_add_iov(iov, iov_len, pkt->virt_hdr.csum_start, csl, cso); 523 524 /* Put the checksum obtained into the packet */ 525 csum = cpu_to_be16(net_checksum_finish_nozero(csum_cntr)); 526 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum); 527 } 528 529 #define NET_MAX_FRAG_SG_LIST (64) 530 531 static size_t net_tx_pkt_fetch_fragment(struct NetTxPkt *pkt, 532 int *src_idx, size_t *src_offset, size_t src_len, 533 struct iovec *dst, int *dst_idx) 534 { 535 size_t fetched = 0; 536 struct iovec *src = pkt->vec; 537 538 while (fetched < src_len) { 539 540 /* no more place in fragment iov */ 541 if (*dst_idx == NET_MAX_FRAG_SG_LIST) { 542 break; 543 } 544 545 /* no more data in iovec */ 546 if (*src_idx == (pkt->payload_frags + NET_TX_PKT_PL_START_FRAG)) { 547 break; 548 } 549 550 551 dst[*dst_idx].iov_base = src[*src_idx].iov_base + *src_offset; 552 dst[*dst_idx].iov_len = MIN(src[*src_idx].iov_len - *src_offset, 553 src_len - fetched); 554 555 *src_offset += dst[*dst_idx].iov_len; 556 fetched += dst[*dst_idx].iov_len; 557 558 if (*src_offset == src[*src_idx].iov_len) { 559 *src_offset = 0; 560 (*src_idx)++; 561 } 562 563 (*dst_idx)++; 564 } 565 566 return fetched; 567 } 568 569 static void net_tx_pkt_sendv( 570 void *opaque, const struct iovec *iov, int iov_cnt, 571 const struct iovec *virt_iov, int virt_iov_cnt) 572 { 573 NetClientState *nc = opaque; 574 575 if (qemu_get_using_vnet_hdr(nc->peer)) { 576 qemu_sendv_packet(nc, virt_iov, virt_iov_cnt); 577 } else { 578 qemu_sendv_packet(nc, iov, iov_cnt); 579 } 580 } 581 582 static bool net_tx_pkt_tcp_fragment_init(struct NetTxPkt *pkt, 583 struct iovec *fragment, 584 int *pl_idx, 585 size_t *l4hdr_len, 586 int *src_idx, 587 size_t *src_offset, 588 size_t *src_len) 589 { 590 struct iovec *l4 = fragment + NET_TX_PKT_PL_START_FRAG; 591 size_t bytes_read = 0; 592 struct tcp_hdr *th; 593 594 if (!pkt->payload_frags) { 595 return false; 596 } 597 598 l4->iov_len = pkt->virt_hdr.hdr_len - pkt->hdr_len; 599 l4->iov_base = g_malloc(l4->iov_len); 600 601 *src_idx = NET_TX_PKT_PL_START_FRAG; 602 while (pkt->vec[*src_idx].iov_len < l4->iov_len - bytes_read) { 603 memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base, 604 pkt->vec[*src_idx].iov_len); 605 606 bytes_read += pkt->vec[*src_idx].iov_len; 607 608 (*src_idx)++; 609 if (*src_idx >= pkt->payload_frags + NET_TX_PKT_PL_START_FRAG) { 610 g_free(l4->iov_base); 611 return false; 612 } 613 } 614 615 *src_offset = l4->iov_len - bytes_read; 616 memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base, 617 *src_offset); 618 619 th = l4->iov_base; 620 th->th_flags &= ~(TH_FIN | TH_PUSH); 621 622 *pl_idx = NET_TX_PKT_PL_START_FRAG + 1; 623 *l4hdr_len = l4->iov_len; 624 *src_len = pkt->virt_hdr.gso_size; 625 626 return true; 627 } 628 629 static void net_tx_pkt_tcp_fragment_deinit(struct iovec *fragment) 630 { 631 g_free(fragment[NET_TX_PKT_PL_START_FRAG].iov_base); 632 } 633 634 static void net_tx_pkt_tcp_fragment_fix(struct NetTxPkt *pkt, 635 struct iovec *fragment, 636 size_t fragment_len, 637 uint8_t gso_type) 638 { 639 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; 640 struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG; 641 struct ip_header *ip = l3hdr->iov_base; 642 struct ip6_header *ip6 = l3hdr->iov_base; 643 size_t len = l3hdr->iov_len + l4hdr->iov_len + fragment_len; 644 645 switch (gso_type) { 646 case VIRTIO_NET_HDR_GSO_TCPV4: 647 ip->ip_len = cpu_to_be16(len); 648 eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len); 649 break; 650 651 case VIRTIO_NET_HDR_GSO_TCPV6: 652 len -= sizeof(struct ip6_header); 653 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = cpu_to_be16(len); 654 break; 655 } 656 } 657 658 static void net_tx_pkt_tcp_fragment_advance(struct NetTxPkt *pkt, 659 struct iovec *fragment, 660 size_t fragment_len, 661 uint8_t gso_type) 662 { 663 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; 664 struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG; 665 struct ip_header *ip = l3hdr->iov_base; 666 struct tcp_hdr *th = l4hdr->iov_base; 667 668 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4) { 669 ip->ip_id = cpu_to_be16(be16_to_cpu(ip->ip_id) + 1); 670 } 671 672 th->th_seq = cpu_to_be32(be32_to_cpu(th->th_seq) + fragment_len); 673 th->th_flags &= ~TH_CWR; 674 } 675 676 static void net_tx_pkt_udp_fragment_init(struct NetTxPkt *pkt, 677 int *pl_idx, 678 size_t *l4hdr_len, 679 int *src_idx, size_t *src_offset, 680 size_t *src_len) 681 { 682 *pl_idx = NET_TX_PKT_PL_START_FRAG; 683 *l4hdr_len = 0; 684 *src_idx = NET_TX_PKT_PL_START_FRAG; 685 *src_offset = 0; 686 *src_len = IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size); 687 } 688 689 static void net_tx_pkt_udp_fragment_fix(struct NetTxPkt *pkt, 690 struct iovec *fragment, 691 size_t fragment_offset, 692 size_t fragment_len) 693 { 694 bool more_frags = fragment_offset + fragment_len < pkt->payload_len; 695 uint16_t orig_flags; 696 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; 697 struct ip_header *ip = l3hdr->iov_base; 698 uint16_t frag_off_units = fragment_offset / IP_FRAG_UNIT_SIZE; 699 uint16_t new_ip_off; 700 701 assert(fragment_offset % IP_FRAG_UNIT_SIZE == 0); 702 assert((frag_off_units & ~IP_OFFMASK) == 0); 703 704 orig_flags = be16_to_cpu(ip->ip_off) & ~(IP_OFFMASK | IP_MF); 705 new_ip_off = frag_off_units | orig_flags | (more_frags ? IP_MF : 0); 706 ip->ip_off = cpu_to_be16(new_ip_off); 707 ip->ip_len = cpu_to_be16(l3hdr->iov_len + fragment_len); 708 709 eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len); 710 } 711 712 static bool net_tx_pkt_do_sw_fragmentation(struct NetTxPkt *pkt, 713 NetTxPktCallback callback, 714 void *context) 715 { 716 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; 717 718 struct iovec fragment[NET_MAX_FRAG_SG_LIST]; 719 size_t fragment_len; 720 size_t l4hdr_len; 721 size_t src_len; 722 723 int src_idx, dst_idx, pl_idx; 724 size_t src_offset; 725 size_t fragment_offset = 0; 726 struct virtio_net_hdr virt_hdr = { 727 .flags = pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM ? 728 VIRTIO_NET_HDR_F_DATA_VALID : 0 729 }; 730 731 /* Copy headers */ 732 fragment[NET_TX_PKT_VHDR_FRAG].iov_base = &virt_hdr; 733 fragment[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof(virt_hdr); 734 fragment[NET_TX_PKT_L2HDR_FRAG] = pkt->vec[NET_TX_PKT_L2HDR_FRAG]; 735 fragment[NET_TX_PKT_L3HDR_FRAG] = pkt->vec[NET_TX_PKT_L3HDR_FRAG]; 736 737 switch (gso_type) { 738 case VIRTIO_NET_HDR_GSO_TCPV4: 739 case VIRTIO_NET_HDR_GSO_TCPV6: 740 if (!net_tx_pkt_tcp_fragment_init(pkt, fragment, &pl_idx, &l4hdr_len, 741 &src_idx, &src_offset, &src_len)) { 742 return false; 743 } 744 break; 745 746 case VIRTIO_NET_HDR_GSO_UDP: 747 net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG], 748 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1, 749 pkt->payload_len); 750 net_tx_pkt_udp_fragment_init(pkt, &pl_idx, &l4hdr_len, 751 &src_idx, &src_offset, &src_len); 752 break; 753 754 default: 755 abort(); 756 } 757 758 /* Put as much data as possible and send */ 759 while (true) { 760 dst_idx = pl_idx; 761 fragment_len = net_tx_pkt_fetch_fragment(pkt, 762 &src_idx, &src_offset, src_len, fragment, &dst_idx); 763 if (!fragment_len) { 764 break; 765 } 766 767 switch (gso_type) { 768 case VIRTIO_NET_HDR_GSO_TCPV4: 769 case VIRTIO_NET_HDR_GSO_TCPV6: 770 net_tx_pkt_tcp_fragment_fix(pkt, fragment, fragment_len, gso_type); 771 net_tx_pkt_do_sw_csum(pkt, fragment + NET_TX_PKT_L2HDR_FRAG, 772 dst_idx - NET_TX_PKT_L2HDR_FRAG, 773 l4hdr_len + fragment_len); 774 break; 775 776 case VIRTIO_NET_HDR_GSO_UDP: 777 net_tx_pkt_udp_fragment_fix(pkt, fragment, fragment_offset, 778 fragment_len); 779 break; 780 } 781 782 callback(context, 783 fragment + NET_TX_PKT_L2HDR_FRAG, dst_idx - NET_TX_PKT_L2HDR_FRAG, 784 fragment + NET_TX_PKT_VHDR_FRAG, dst_idx - NET_TX_PKT_VHDR_FRAG); 785 786 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 787 gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 788 net_tx_pkt_tcp_fragment_advance(pkt, fragment, fragment_len, 789 gso_type); 790 } 791 792 fragment_offset += fragment_len; 793 } 794 795 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 796 gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 797 net_tx_pkt_tcp_fragment_deinit(fragment); 798 } 799 800 return true; 801 } 802 803 bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc) 804 { 805 bool offload = qemu_get_using_vnet_hdr(nc->peer); 806 return net_tx_pkt_send_custom(pkt, offload, net_tx_pkt_sendv, nc); 807 } 808 809 bool net_tx_pkt_send_custom(struct NetTxPkt *pkt, bool offload, 810 NetTxPktCallback callback, void *context) 811 { 812 assert(pkt); 813 814 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; 815 816 /* 817 * Since underlying infrastructure does not support IP datagrams longer 818 * than 64K we should drop such packets and don't even try to send 819 */ 820 if (VIRTIO_NET_HDR_GSO_NONE != gso_type) { 821 if (pkt->payload_len > 822 ETH_MAX_IP_DGRAM_LEN - 823 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len) { 824 return false; 825 } 826 } 827 828 if (offload || gso_type == VIRTIO_NET_HDR_GSO_NONE) { 829 if (!offload && pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 830 net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG], 831 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1, 832 pkt->payload_len); 833 } 834 835 net_tx_pkt_fix_ip6_payload_len(pkt); 836 callback(context, pkt->vec + NET_TX_PKT_L2HDR_FRAG, 837 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_L2HDR_FRAG, 838 pkt->vec + NET_TX_PKT_VHDR_FRAG, 839 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_VHDR_FRAG); 840 return true; 841 } 842 843 return net_tx_pkt_do_sw_fragmentation(pkt, callback, context); 844 } 845 846 void net_tx_pkt_fix_ip6_payload_len(struct NetTxPkt *pkt) 847 { 848 struct iovec *l2 = &pkt->vec[NET_TX_PKT_L2HDR_FRAG]; 849 if (eth_get_l3_proto(l2, 1, l2->iov_len) == ETH_P_IPV6) { 850 /* 851 * TODO: if qemu would support >64K packets - add jumbo option check 852 * something like that: 853 * 'if (ip6->ip6_plen == 0 && !has_jumbo_option(ip6)) {' 854 */ 855 if (pkt->l3_hdr.ip6.ip6_plen == 0) { 856 if (pkt->payload_len <= ETH_MAX_IP_DGRAM_LEN) { 857 pkt->l3_hdr.ip6.ip6_plen = htons(pkt->payload_len); 858 } 859 /* 860 * TODO: if qemu would support >64K packets 861 * add jumbo option for packets greater then 65,535 bytes 862 */ 863 } 864 } 865 } 866