1 /* 2 * QEMU NVM Express End-to-End Data Protection support 3 * 4 * Copyright (c) 2021 Samsung Electronics Co., Ltd. 5 * 6 * Authors: 7 * Klaus Jensen <k.jensen@samsung.com> 8 * Gollu Appalanaidu <anaidu.gollu@samsung.com> 9 */ 10 11 #include "qemu/osdep.h" 12 #include "qapi/error.h" 13 #include "system/block-backend.h" 14 15 #include "nvme.h" 16 #include "dif.h" 17 #include "trace.h" 18 19 uint16_t nvme_check_prinfo(NvmeNamespace *ns, uint8_t prinfo, uint64_t slba, 20 uint64_t reftag) 21 { 22 uint64_t mask = ns->pif ? 0xffffffffffff : 0xffffffff; 23 24 if ((NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) == NVME_ID_NS_DPS_TYPE_1) && 25 (prinfo & NVME_PRINFO_PRCHK_REF) && (slba & mask) != reftag) { 26 return NVME_INVALID_PROT_INFO | NVME_DNR; 27 } 28 29 if ((NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) == NVME_ID_NS_DPS_TYPE_3) && 30 (prinfo & NVME_PRINFO_PRCHK_REF)) { 31 return NVME_INVALID_PROT_INFO; 32 } 33 34 return NVME_SUCCESS; 35 } 36 37 /* from Linux kernel (crypto/crct10dif_common.c) */ 38 static uint16_t crc16_t10dif(uint16_t crc, const unsigned char *buffer, 39 size_t len) 40 { 41 unsigned int i; 42 43 for (i = 0; i < len; i++) { 44 crc = (crc << 8) ^ crc16_t10dif_table[((crc >> 8) ^ buffer[i]) & 0xff]; 45 } 46 47 return crc; 48 } 49 50 /* from Linux kernel (lib/crc64.c) */ 51 static uint64_t crc64_nvme(uint64_t crc, const unsigned char *buffer, 52 size_t len) 53 { 54 size_t i; 55 56 for (i = 0; i < len; i++) { 57 crc = (crc >> 8) ^ crc64_nvme_table[(crc & 0xff) ^ buffer[i]]; 58 } 59 60 return crc ^ (uint64_t)~0; 61 } 62 63 static void nvme_dif_pract_generate_dif_crc16(NvmeNamespace *ns, uint8_t *buf, 64 size_t len, uint8_t *mbuf, 65 size_t mlen, uint16_t apptag, 66 uint64_t *reftag) 67 { 68 uint8_t *end = buf + len; 69 int16_t pil = 0; 70 71 if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) { 72 pil = ns->lbaf.ms - nvme_pi_tuple_size(ns); 73 } 74 75 trace_pci_nvme_dif_pract_generate_dif_crc16(len, ns->lbasz, 76 ns->lbasz + pil, apptag, 77 *reftag); 78 79 for (; buf < end; buf += ns->lbasz, mbuf += ns->lbaf.ms) { 80 NvmeDifTuple *dif = (NvmeDifTuple *)(mbuf + pil); 81 uint16_t crc = crc16_t10dif(0x0, buf, ns->lbasz); 82 83 if (pil) { 84 crc = crc16_t10dif(crc, mbuf, pil); 85 } 86 87 dif->g16.guard = cpu_to_be16(crc); 88 dif->g16.apptag = cpu_to_be16(apptag); 89 dif->g16.reftag = cpu_to_be32(*reftag); 90 91 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) != NVME_ID_NS_DPS_TYPE_3) { 92 (*reftag)++; 93 } 94 } 95 } 96 97 static void nvme_dif_pract_generate_dif_crc64(NvmeNamespace *ns, uint8_t *buf, 98 size_t len, uint8_t *mbuf, 99 size_t mlen, uint16_t apptag, 100 uint64_t *reftag) 101 { 102 uint8_t *end = buf + len; 103 int16_t pil = 0; 104 105 if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) { 106 pil = ns->lbaf.ms - 16; 107 } 108 109 trace_pci_nvme_dif_pract_generate_dif_crc64(len, ns->lbasz, 110 ns->lbasz + pil, apptag, 111 *reftag); 112 113 for (; buf < end; buf += ns->lbasz, mbuf += ns->lbaf.ms) { 114 NvmeDifTuple *dif = (NvmeDifTuple *)(mbuf + pil); 115 uint64_t crc = crc64_nvme(~0ULL, buf, ns->lbasz); 116 117 if (pil) { 118 crc = crc64_nvme(~crc, mbuf, pil); 119 } 120 121 dif->g64.guard = cpu_to_be64(crc); 122 dif->g64.apptag = cpu_to_be16(apptag); 123 124 dif->g64.sr[0] = *reftag >> 40; 125 dif->g64.sr[1] = *reftag >> 32; 126 dif->g64.sr[2] = *reftag >> 24; 127 dif->g64.sr[3] = *reftag >> 16; 128 dif->g64.sr[4] = *reftag >> 8; 129 dif->g64.sr[5] = *reftag; 130 131 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) != NVME_ID_NS_DPS_TYPE_3) { 132 (*reftag)++; 133 } 134 } 135 } 136 137 void nvme_dif_pract_generate_dif(NvmeNamespace *ns, uint8_t *buf, size_t len, 138 uint8_t *mbuf, size_t mlen, uint16_t apptag, 139 uint64_t *reftag) 140 { 141 switch (ns->pif) { 142 case NVME_PI_GUARD_16: 143 return nvme_dif_pract_generate_dif_crc16(ns, buf, len, mbuf, mlen, 144 apptag, reftag); 145 case NVME_PI_GUARD_64: 146 return nvme_dif_pract_generate_dif_crc64(ns, buf, len, mbuf, mlen, 147 apptag, reftag); 148 } 149 150 abort(); 151 } 152 153 static uint16_t nvme_dif_prchk_crc16(NvmeNamespace *ns, NvmeDifTuple *dif, 154 uint8_t *buf, uint8_t *mbuf, size_t pil, 155 uint8_t prinfo, uint16_t apptag, 156 uint16_t appmask, uint64_t reftag) 157 { 158 switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { 159 case NVME_ID_NS_DPS_TYPE_3: 160 if (be32_to_cpu(dif->g16.reftag) != 0xffffffff) { 161 break; 162 } 163 164 /* fallthrough */ 165 case NVME_ID_NS_DPS_TYPE_1: 166 case NVME_ID_NS_DPS_TYPE_2: 167 if (be16_to_cpu(dif->g16.apptag) != 0xffff) { 168 break; 169 } 170 171 trace_pci_nvme_dif_prchk_disabled_crc16(be16_to_cpu(dif->g16.apptag), 172 be32_to_cpu(dif->g16.reftag)); 173 174 return NVME_SUCCESS; 175 } 176 177 if (prinfo & NVME_PRINFO_PRCHK_GUARD) { 178 uint16_t crc = crc16_t10dif(0x0, buf, ns->lbasz); 179 180 if (pil) { 181 crc = crc16_t10dif(crc, mbuf, pil); 182 } 183 184 trace_pci_nvme_dif_prchk_guard_crc16(be16_to_cpu(dif->g16.guard), crc); 185 186 if (be16_to_cpu(dif->g16.guard) != crc) { 187 return NVME_E2E_GUARD_ERROR; 188 } 189 } 190 191 if (prinfo & NVME_PRINFO_PRCHK_APP) { 192 trace_pci_nvme_dif_prchk_apptag(be16_to_cpu(dif->g16.apptag), apptag, 193 appmask); 194 195 if ((be16_to_cpu(dif->g16.apptag) & appmask) != (apptag & appmask)) { 196 return NVME_E2E_APP_ERROR; 197 } 198 } 199 200 if (prinfo & NVME_PRINFO_PRCHK_REF) { 201 trace_pci_nvme_dif_prchk_reftag_crc16(be32_to_cpu(dif->g16.reftag), 202 reftag); 203 204 if (be32_to_cpu(dif->g16.reftag) != reftag) { 205 return NVME_E2E_REF_ERROR; 206 } 207 } 208 209 return NVME_SUCCESS; 210 } 211 212 static uint16_t nvme_dif_prchk_crc64(NvmeNamespace *ns, NvmeDifTuple *dif, 213 uint8_t *buf, uint8_t *mbuf, size_t pil, 214 uint8_t prinfo, uint16_t apptag, 215 uint16_t appmask, uint64_t reftag) 216 { 217 uint64_t r = 0; 218 219 r |= (uint64_t)dif->g64.sr[0] << 40; 220 r |= (uint64_t)dif->g64.sr[1] << 32; 221 r |= (uint64_t)dif->g64.sr[2] << 24; 222 r |= (uint64_t)dif->g64.sr[3] << 16; 223 r |= (uint64_t)dif->g64.sr[4] << 8; 224 r |= (uint64_t)dif->g64.sr[5]; 225 226 switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { 227 case NVME_ID_NS_DPS_TYPE_3: 228 if (r != 0xffffffffffff) { 229 break; 230 } 231 232 /* fallthrough */ 233 case NVME_ID_NS_DPS_TYPE_1: 234 case NVME_ID_NS_DPS_TYPE_2: 235 if (be16_to_cpu(dif->g64.apptag) != 0xffff) { 236 break; 237 } 238 239 trace_pci_nvme_dif_prchk_disabled_crc64(be16_to_cpu(dif->g16.apptag), 240 r); 241 242 return NVME_SUCCESS; 243 } 244 245 if (prinfo & NVME_PRINFO_PRCHK_GUARD) { 246 uint64_t crc = crc64_nvme(~0ULL, buf, ns->lbasz); 247 248 if (pil) { 249 crc = crc64_nvme(~crc, mbuf, pil); 250 } 251 252 trace_pci_nvme_dif_prchk_guard_crc64(be64_to_cpu(dif->g64.guard), crc); 253 254 if (be64_to_cpu(dif->g64.guard) != crc) { 255 return NVME_E2E_GUARD_ERROR; 256 } 257 } 258 259 if (prinfo & NVME_PRINFO_PRCHK_APP) { 260 trace_pci_nvme_dif_prchk_apptag(be16_to_cpu(dif->g64.apptag), apptag, 261 appmask); 262 263 if ((be16_to_cpu(dif->g64.apptag) & appmask) != (apptag & appmask)) { 264 return NVME_E2E_APP_ERROR; 265 } 266 } 267 268 if (prinfo & NVME_PRINFO_PRCHK_REF) { 269 trace_pci_nvme_dif_prchk_reftag_crc64(r, reftag); 270 271 if (r != reftag) { 272 return NVME_E2E_REF_ERROR; 273 } 274 } 275 276 return NVME_SUCCESS; 277 } 278 279 static uint16_t nvme_dif_prchk(NvmeNamespace *ns, NvmeDifTuple *dif, 280 uint8_t *buf, uint8_t *mbuf, size_t pil, 281 uint8_t prinfo, uint16_t apptag, 282 uint16_t appmask, uint64_t reftag) 283 { 284 switch (ns->pif) { 285 case NVME_PI_GUARD_16: 286 return nvme_dif_prchk_crc16(ns, dif, buf, mbuf, pil, prinfo, apptag, 287 appmask, reftag); 288 case NVME_PI_GUARD_64: 289 return nvme_dif_prchk_crc64(ns, dif, buf, mbuf, pil, prinfo, apptag, 290 appmask, reftag); 291 } 292 293 abort(); 294 } 295 296 uint16_t nvme_dif_check(NvmeNamespace *ns, uint8_t *buf, size_t len, 297 uint8_t *mbuf, size_t mlen, uint8_t prinfo, 298 uint64_t slba, uint16_t apptag, 299 uint16_t appmask, uint64_t *reftag) 300 { 301 uint8_t *bufp, *end = buf + len; 302 int16_t pil = 0; 303 uint16_t status; 304 305 status = nvme_check_prinfo(ns, prinfo, slba, *reftag); 306 if (status) { 307 return status; 308 } 309 310 if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) { 311 pil = ns->lbaf.ms - nvme_pi_tuple_size(ns); 312 } 313 314 trace_pci_nvme_dif_check(prinfo, ns->lbasz + pil); 315 316 for (bufp = buf; bufp < end; bufp += ns->lbasz, mbuf += ns->lbaf.ms) { 317 NvmeDifTuple *dif = (NvmeDifTuple *)(mbuf + pil); 318 status = nvme_dif_prchk(ns, dif, bufp, mbuf, pil, prinfo, apptag, 319 appmask, *reftag); 320 if (status) { 321 /* 322 * The first block of a 'raw' image is always allocated, so we 323 * cannot reliably know if the block is all zeroes or not. For 324 * CRC16 this works fine because the T10 CRC16 is 0x0 for all 325 * zeroes, but the Rocksoft CRC64 is not. Thus, if a guard error is 326 * detected for the first block, check if it is zeroed and manually 327 * set the protection information to all ones to disable protection 328 * information checking. 329 */ 330 if (status == NVME_E2E_GUARD_ERROR && slba == 0x0 && bufp == buf) { 331 g_autofree uint8_t *zeroes = g_malloc0(ns->lbasz); 332 333 if (memcmp(bufp, zeroes, ns->lbasz) == 0) { 334 memset(mbuf + pil, 0xff, nvme_pi_tuple_size(ns)); 335 } 336 } else { 337 return status; 338 } 339 } 340 341 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) != NVME_ID_NS_DPS_TYPE_3) { 342 (*reftag)++; 343 } 344 } 345 346 return NVME_SUCCESS; 347 } 348 349 uint16_t nvme_dif_mangle_mdata(NvmeNamespace *ns, uint8_t *mbuf, size_t mlen, 350 uint64_t slba) 351 { 352 BlockBackend *blk = ns->blkconf.blk; 353 BlockDriverState *bs = blk_bs(blk); 354 355 int64_t moffset = 0, offset = nvme_l2b(ns, slba); 356 uint8_t *mbufp, *end; 357 bool zeroed; 358 int16_t pil = 0; 359 int64_t bytes = (mlen / ns->lbaf.ms) << ns->lbaf.ds; 360 int64_t pnum = 0; 361 362 Error *err = NULL; 363 364 365 if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) { 366 pil = ns->lbaf.ms - nvme_pi_tuple_size(ns); 367 } 368 369 do { 370 int ret; 371 372 bytes -= pnum; 373 374 ret = bdrv_block_status(bs, offset, bytes, &pnum, NULL, NULL); 375 if (ret < 0) { 376 error_setg_errno(&err, -ret, "unable to get block status"); 377 error_report_err(err); 378 379 return NVME_INTERNAL_DEV_ERROR; 380 } 381 382 zeroed = !!(ret & BDRV_BLOCK_ZERO); 383 384 trace_pci_nvme_block_status(offset, bytes, pnum, ret, zeroed); 385 386 if (zeroed) { 387 mbufp = mbuf + moffset; 388 mlen = (pnum >> ns->lbaf.ds) * ns->lbaf.ms; 389 end = mbufp + mlen; 390 391 for (; mbufp < end; mbufp += ns->lbaf.ms) { 392 memset(mbufp + pil, 0xff, nvme_pi_tuple_size(ns)); 393 } 394 } 395 396 moffset += (pnum >> ns->lbaf.ds) * ns->lbaf.ms; 397 offset += pnum; 398 } while (pnum != bytes); 399 400 return NVME_SUCCESS; 401 } 402 403 static void nvme_dif_rw_cb(void *opaque, int ret) 404 { 405 NvmeBounceContext *ctx = opaque; 406 NvmeRequest *req = ctx->req; 407 NvmeNamespace *ns = req->ns; 408 BlockBackend *blk = ns->blkconf.blk; 409 410 trace_pci_nvme_dif_rw_cb(nvme_cid(req), blk_name(blk)); 411 412 qemu_iovec_destroy(&ctx->data.iov); 413 g_free(ctx->data.bounce); 414 415 qemu_iovec_destroy(&ctx->mdata.iov); 416 g_free(ctx->mdata.bounce); 417 418 g_free(ctx); 419 420 nvme_rw_complete_cb(req, ret); 421 } 422 423 static void nvme_dif_rw_check_cb(void *opaque, int ret) 424 { 425 NvmeBounceContext *ctx = opaque; 426 NvmeRequest *req = ctx->req; 427 NvmeNamespace *ns = req->ns; 428 NvmeCtrl *n = nvme_ctrl(req); 429 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; 430 uint64_t slba = le64_to_cpu(rw->slba); 431 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control)); 432 uint16_t apptag = le16_to_cpu(rw->apptag); 433 uint16_t appmask = le16_to_cpu(rw->appmask); 434 uint64_t reftag = le32_to_cpu(rw->reftag); 435 uint64_t cdw3 = le32_to_cpu(rw->cdw3); 436 uint16_t status; 437 438 reftag |= cdw3 << 32; 439 440 trace_pci_nvme_dif_rw_check_cb(nvme_cid(req), prinfo, apptag, appmask, 441 reftag); 442 443 if (ret) { 444 goto out; 445 } 446 447 status = nvme_dif_mangle_mdata(ns, ctx->mdata.bounce, ctx->mdata.iov.size, 448 slba); 449 if (status) { 450 req->status = status; 451 goto out; 452 } 453 454 status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size, 455 ctx->mdata.bounce, ctx->mdata.iov.size, prinfo, 456 slba, apptag, appmask, &reftag); 457 if (status) { 458 req->status = status; 459 goto out; 460 } 461 462 status = nvme_bounce_data(n, ctx->data.bounce, ctx->data.iov.size, 463 NVME_TX_DIRECTION_FROM_DEVICE, req); 464 if (status) { 465 req->status = status; 466 goto out; 467 } 468 469 if (prinfo & NVME_PRINFO_PRACT && ns->lbaf.ms == nvme_pi_tuple_size(ns)) { 470 goto out; 471 } 472 473 status = nvme_bounce_mdata(n, ctx->mdata.bounce, ctx->mdata.iov.size, 474 NVME_TX_DIRECTION_FROM_DEVICE, req); 475 if (status) { 476 req->status = status; 477 } 478 479 out: 480 nvme_dif_rw_cb(ctx, ret); 481 } 482 483 static void nvme_dif_rw_mdata_in_cb(void *opaque, int ret) 484 { 485 NvmeBounceContext *ctx = opaque; 486 NvmeRequest *req = ctx->req; 487 NvmeNamespace *ns = req->ns; 488 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; 489 uint64_t slba = le64_to_cpu(rw->slba); 490 uint32_t nlb = le16_to_cpu(rw->nlb) + 1; 491 size_t mlen = nvme_m2b(ns, nlb); 492 uint64_t offset = nvme_moff(ns, slba); 493 BlockBackend *blk = ns->blkconf.blk; 494 495 trace_pci_nvme_dif_rw_mdata_in_cb(nvme_cid(req), blk_name(blk)); 496 497 if (ret) { 498 goto out; 499 } 500 501 ctx->mdata.bounce = g_malloc(mlen); 502 503 qemu_iovec_reset(&ctx->mdata.iov); 504 qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen); 505 506 req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0, 507 nvme_dif_rw_check_cb, ctx); 508 return; 509 510 out: 511 nvme_dif_rw_cb(ctx, ret); 512 } 513 514 static void nvme_dif_rw_mdata_out_cb(void *opaque, int ret) 515 { 516 NvmeBounceContext *ctx = opaque; 517 NvmeRequest *req = ctx->req; 518 NvmeNamespace *ns = req->ns; 519 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; 520 uint64_t slba = le64_to_cpu(rw->slba); 521 uint64_t offset = nvme_moff(ns, slba); 522 BlockBackend *blk = ns->blkconf.blk; 523 524 trace_pci_nvme_dif_rw_mdata_out_cb(nvme_cid(req), blk_name(blk)); 525 526 if (ret) { 527 goto out; 528 } 529 530 req->aiocb = blk_aio_pwritev(blk, offset, &ctx->mdata.iov, 0, 531 nvme_dif_rw_cb, ctx); 532 return; 533 534 out: 535 nvme_dif_rw_cb(ctx, ret); 536 } 537 538 uint16_t nvme_dif_rw(NvmeCtrl *n, NvmeRequest *req) 539 { 540 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; 541 NvmeNamespace *ns = req->ns; 542 BlockBackend *blk = ns->blkconf.blk; 543 bool wrz = rw->opcode == NVME_CMD_WRITE_ZEROES; 544 uint32_t nlb = le16_to_cpu(rw->nlb) + 1; 545 uint64_t slba = le64_to_cpu(rw->slba); 546 size_t len = nvme_l2b(ns, nlb); 547 size_t mlen = nvme_m2b(ns, nlb); 548 size_t mapped_len = len; 549 int64_t offset = nvme_l2b(ns, slba); 550 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control)); 551 uint16_t apptag = le16_to_cpu(rw->apptag); 552 uint16_t appmask = le16_to_cpu(rw->appmask); 553 uint64_t reftag = le32_to_cpu(rw->reftag); 554 uint64_t cdw3 = le32_to_cpu(rw->cdw3); 555 bool pract = !!(prinfo & NVME_PRINFO_PRACT); 556 NvmeBounceContext *ctx; 557 uint16_t status; 558 559 reftag |= cdw3 << 32; 560 561 trace_pci_nvme_dif_rw(pract, prinfo); 562 563 ctx = g_new0(NvmeBounceContext, 1); 564 ctx->req = req; 565 566 if (wrz) { 567 BdrvRequestFlags flags = BDRV_REQ_MAY_UNMAP; 568 569 if (prinfo & NVME_PRINFO_PRCHK_MASK) { 570 status = NVME_INVALID_PROT_INFO | NVME_DNR; 571 goto err; 572 } 573 574 if (pract) { 575 uint8_t *mbuf, *end; 576 int16_t pil = ns->lbaf.ms - nvme_pi_tuple_size(ns); 577 578 flags = 0; 579 580 ctx->mdata.bounce = g_malloc0(mlen); 581 582 qemu_iovec_init(&ctx->mdata.iov, 1); 583 qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen); 584 585 mbuf = ctx->mdata.bounce; 586 end = mbuf + mlen; 587 588 if (ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT) { 589 pil = 0; 590 } 591 592 for (; mbuf < end; mbuf += ns->lbaf.ms) { 593 NvmeDifTuple *dif = (NvmeDifTuple *)(mbuf + pil); 594 595 switch (ns->pif) { 596 case NVME_PI_GUARD_16: 597 dif->g16.apptag = cpu_to_be16(apptag); 598 dif->g16.reftag = cpu_to_be32(reftag); 599 600 break; 601 602 case NVME_PI_GUARD_64: 603 dif->g64.guard = cpu_to_be64(0x6482d367eb22b64e); 604 dif->g64.apptag = cpu_to_be16(apptag); 605 606 dif->g64.sr[0] = reftag >> 40; 607 dif->g64.sr[1] = reftag >> 32; 608 dif->g64.sr[2] = reftag >> 24; 609 dif->g64.sr[3] = reftag >> 16; 610 dif->g64.sr[4] = reftag >> 8; 611 dif->g64.sr[5] = reftag; 612 613 break; 614 615 default: 616 abort(); 617 } 618 619 switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { 620 case NVME_ID_NS_DPS_TYPE_1: 621 case NVME_ID_NS_DPS_TYPE_2: 622 reftag++; 623 } 624 } 625 } 626 627 req->aiocb = blk_aio_pwrite_zeroes(blk, offset, len, flags, 628 nvme_dif_rw_mdata_out_cb, ctx); 629 return NVME_NO_COMPLETE; 630 } 631 632 if (nvme_ns_ext(ns) && !(pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) { 633 mapped_len += mlen; 634 } 635 636 status = nvme_map_dptr(n, &req->sg, mapped_len, &req->cmd); 637 if (status) { 638 goto err; 639 } 640 641 ctx->data.bounce = g_malloc(len); 642 643 qemu_iovec_init(&ctx->data.iov, 1); 644 qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, len); 645 646 if (req->cmd.opcode == NVME_CMD_READ) { 647 block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size, 648 BLOCK_ACCT_READ); 649 650 req->aiocb = blk_aio_preadv(ns->blkconf.blk, offset, &ctx->data.iov, 0, 651 nvme_dif_rw_mdata_in_cb, ctx); 652 return NVME_NO_COMPLETE; 653 } 654 655 status = nvme_bounce_data(n, ctx->data.bounce, ctx->data.iov.size, 656 NVME_TX_DIRECTION_TO_DEVICE, req); 657 if (status) { 658 goto err; 659 } 660 661 ctx->mdata.bounce = g_malloc(mlen); 662 663 qemu_iovec_init(&ctx->mdata.iov, 1); 664 qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen); 665 666 if (!(pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) { 667 status = nvme_bounce_mdata(n, ctx->mdata.bounce, ctx->mdata.iov.size, 668 NVME_TX_DIRECTION_TO_DEVICE, req); 669 if (status) { 670 goto err; 671 } 672 } 673 674 status = nvme_check_prinfo(ns, prinfo, slba, reftag); 675 if (status) { 676 goto err; 677 } 678 679 if (pract) { 680 /* splice generated protection information into the buffer */ 681 nvme_dif_pract_generate_dif(ns, ctx->data.bounce, ctx->data.iov.size, 682 ctx->mdata.bounce, ctx->mdata.iov.size, 683 apptag, &reftag); 684 } else { 685 status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size, 686 ctx->mdata.bounce, ctx->mdata.iov.size, prinfo, 687 slba, apptag, appmask, &reftag); 688 if (status) { 689 goto err; 690 } 691 } 692 693 block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size, 694 BLOCK_ACCT_WRITE); 695 696 req->aiocb = blk_aio_pwritev(ns->blkconf.blk, offset, &ctx->data.iov, 0, 697 nvme_dif_rw_mdata_out_cb, ctx); 698 699 return NVME_NO_COMPLETE; 700 701 err: 702 qemu_iovec_destroy(&ctx->data.iov); 703 g_free(ctx->data.bounce); 704 705 qemu_iovec_destroy(&ctx->mdata.iov); 706 g_free(ctx->mdata.bounce); 707 708 g_free(ctx); 709 710 return status; 711 } 712