1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Passthrough DMA device driver 4 * -- Based on the CCP driver 5 * 6 * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. 7 * 8 * Author: Sanjay R Mehta <sanju.mehta@amd.com> 9 * Author: Gary R Hook <gary.hook@amd.com> 10 */ 11 12 #include <linux/bitfield.h> 13 #include "ptdma.h" 14 #include "../ae4dma/ae4dma.h" 15 #include "../../dmaengine.h" 16 17 static char *ae4_error_codes[] = { 18 "", 19 "ERR 01: INVALID HEADER DW0", 20 "ERR 02: INVALID STATUS", 21 "ERR 03: INVALID LENGTH - 4 BYTE ALIGNMENT", 22 "ERR 04: INVALID SRC ADDR - 4 BYTE ALIGNMENT", 23 "ERR 05: INVALID DST ADDR - 4 BYTE ALIGNMENT", 24 "ERR 06: INVALID ALIGNMENT", 25 "ERR 07: INVALID DESCRIPTOR", 26 }; 27 28 static void ae4_log_error(struct pt_device *d, int e) 29 { 30 /* ERR 01 - 07 represents Invalid AE4 errors */ 31 if (e <= 7) 32 dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", ae4_error_codes[e], e); 33 /* ERR 08 - 15 represents Invalid Descriptor errors */ 34 else if (e > 7 && e <= 15) 35 dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "INVALID DESCRIPTOR", e); 36 /* ERR 16 - 31 represents Firmware errors */ 37 else if (e > 15 && e <= 31) 38 dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "FIRMWARE ERROR", e); 39 /* ERR 32 - 63 represents Fatal errors */ 40 else if (e > 31 && e <= 63) 41 dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "FATAL ERROR", e); 42 /* ERR 64 - 255 represents PTE errors */ 43 else if (e > 63 && e <= 255) 44 dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "PTE ERROR", e); 45 else 46 dev_info(d->dev, "Unknown AE4DMA error"); 47 } 48 49 void ae4_check_status_error(struct ae4_cmd_queue *ae4cmd_q, int idx) 50 { 51 struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; 52 struct ae4dma_desc desc; 53 u8 status; 54 55 memcpy(&desc, &cmd_q->qbase[idx], sizeof(struct ae4dma_desc)); 56 status = desc.dw1.status; 57 if (status && status != AE4_DESC_COMPLETED) { 58 cmd_q->cmd_error = desc.dw1.err_code; 59 if (cmd_q->cmd_error) 60 ae4_log_error(cmd_q->pt, cmd_q->cmd_error); 61 } 62 } 63 EXPORT_SYMBOL_GPL(ae4_check_status_error); 64 65 static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan) 66 { 67 return container_of(dma_chan, struct pt_dma_chan, vc.chan); 68 } 69 70 static inline struct pt_dma_desc *to_pt_desc(struct virt_dma_desc *vd) 71 { 72 return container_of(vd, struct pt_dma_desc, vd); 73 } 74 75 static void pt_free_chan_resources(struct dma_chan *dma_chan) 76 { 77 struct pt_dma_chan *chan = to_pt_chan(dma_chan); 78 79 vchan_free_chan_resources(&chan->vc); 80 } 81 82 static void pt_synchronize(struct dma_chan *dma_chan) 83 { 84 struct pt_dma_chan *chan = to_pt_chan(dma_chan); 85 86 vchan_synchronize(&chan->vc); 87 } 88 89 static void pt_do_cleanup(struct virt_dma_desc *vd) 90 { 91 struct pt_dma_desc *desc = to_pt_desc(vd); 92 struct pt_device *pt = desc->pt; 93 94 kmem_cache_free(pt->dma_desc_cache, desc); 95 } 96 97 static struct pt_cmd_queue *pt_get_cmd_queue(struct pt_device *pt, struct pt_dma_chan *chan) 98 { 99 struct ae4_cmd_queue *ae4cmd_q; 100 struct pt_cmd_queue *cmd_q; 101 struct ae4_device *ae4; 102 103 if (pt->ver == AE4_DMA_VERSION) { 104 ae4 = container_of(pt, struct ae4_device, pt); 105 ae4cmd_q = &ae4->ae4cmd_q[chan->id]; 106 cmd_q = &ae4cmd_q->cmd_q; 107 } else { 108 cmd_q = &pt->cmd_q; 109 } 110 111 return cmd_q; 112 } 113 114 static int ae4_core_execute_cmd(struct ae4dma_desc *desc, struct ae4_cmd_queue *ae4cmd_q) 115 { 116 bool soc = FIELD_GET(DWORD0_SOC, desc->dwouv.dw0); 117 struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; 118 119 if (soc) { 120 desc->dwouv.dw0 |= FIELD_PREP(DWORD0_IOC, desc->dwouv.dw0); 121 desc->dwouv.dw0 &= ~DWORD0_SOC; 122 } 123 124 mutex_lock(&ae4cmd_q->cmd_lock); 125 memcpy(&cmd_q->qbase[ae4cmd_q->tail_wi], desc, sizeof(struct ae4dma_desc)); 126 ae4cmd_q->q_cmd_count++; 127 ae4cmd_q->tail_wi = (ae4cmd_q->tail_wi + 1) % CMD_Q_LEN; 128 writel(ae4cmd_q->tail_wi, cmd_q->reg_control + AE4_WR_IDX_OFF); 129 mutex_unlock(&ae4cmd_q->cmd_lock); 130 131 wake_up(&ae4cmd_q->q_w); 132 133 return 0; 134 } 135 136 static int pt_core_perform_passthru_ae4(struct pt_cmd_queue *cmd_q, 137 struct pt_passthru_engine *pt_engine) 138 { 139 struct ae4_cmd_queue *ae4cmd_q = container_of(cmd_q, struct ae4_cmd_queue, cmd_q); 140 struct ae4dma_desc desc; 141 142 cmd_q->cmd_error = 0; 143 cmd_q->total_pt_ops++; 144 memset(&desc, 0, sizeof(desc)); 145 desc.dwouv.dws.byte0 = CMD_AE4_DESC_DW0_VAL; 146 147 desc.dw1.status = 0; 148 desc.dw1.err_code = 0; 149 desc.dw1.desc_id = 0; 150 151 desc.length = pt_engine->src_len; 152 153 desc.src_lo = upper_32_bits(pt_engine->src_dma); 154 desc.src_hi = lower_32_bits(pt_engine->src_dma); 155 desc.dst_lo = upper_32_bits(pt_engine->dst_dma); 156 desc.dst_hi = lower_32_bits(pt_engine->dst_dma); 157 158 return ae4_core_execute_cmd(&desc, ae4cmd_q); 159 } 160 161 static int pt_dma_start_desc(struct pt_dma_desc *desc, struct pt_dma_chan *chan) 162 { 163 struct pt_passthru_engine *pt_engine; 164 struct pt_device *pt; 165 struct pt_cmd *pt_cmd; 166 struct pt_cmd_queue *cmd_q; 167 168 desc->issued_to_hw = 1; 169 170 pt_cmd = &desc->pt_cmd; 171 pt = pt_cmd->pt; 172 173 cmd_q = pt_get_cmd_queue(pt, chan); 174 175 pt_engine = &pt_cmd->passthru; 176 177 pt->tdata.cmd = pt_cmd; 178 179 /* Execute the command */ 180 if (pt->ver == AE4_DMA_VERSION) 181 pt_cmd->ret = pt_core_perform_passthru_ae4(cmd_q, pt_engine); 182 else 183 pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine); 184 185 return 0; 186 } 187 188 static struct pt_dma_desc *pt_next_dma_desc(struct pt_dma_chan *chan) 189 { 190 /* Get the next DMA descriptor on the active list */ 191 struct virt_dma_desc *vd = vchan_next_desc(&chan->vc); 192 193 return vd ? to_pt_desc(vd) : NULL; 194 } 195 196 static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan, 197 struct pt_dma_desc *desc) 198 { 199 struct dma_async_tx_descriptor *tx_desc; 200 struct virt_dma_desc *vd; 201 struct pt_device *pt; 202 unsigned long flags; 203 204 pt = chan->pt; 205 /* Loop over descriptors until one is found with commands */ 206 do { 207 if (desc) { 208 if (!desc->issued_to_hw) { 209 /* No errors, keep going */ 210 if (desc->status != DMA_ERROR) 211 return desc; 212 } 213 214 tx_desc = &desc->vd.tx; 215 vd = &desc->vd; 216 } else { 217 tx_desc = NULL; 218 } 219 220 spin_lock_irqsave(&chan->vc.lock, flags); 221 222 if (pt->ver != AE4_DMA_VERSION && desc) { 223 if (desc->status != DMA_COMPLETE) { 224 if (desc->status != DMA_ERROR) 225 desc->status = DMA_COMPLETE; 226 227 dma_cookie_complete(tx_desc); 228 dma_descriptor_unmap(tx_desc); 229 list_del(&desc->vd.node); 230 } else { 231 /* Don't handle it twice */ 232 tx_desc = NULL; 233 } 234 } 235 236 desc = pt_next_dma_desc(chan); 237 238 spin_unlock_irqrestore(&chan->vc.lock, flags); 239 240 if (pt->ver != AE4_DMA_VERSION && tx_desc) { 241 dmaengine_desc_get_callback_invoke(tx_desc, NULL); 242 dma_run_dependencies(tx_desc); 243 vchan_vdesc_fini(vd); 244 } 245 } while (desc); 246 247 return NULL; 248 } 249 250 static inline bool ae4_core_queue_full(struct pt_cmd_queue *cmd_q) 251 { 252 u32 front_wi = readl(cmd_q->reg_control + AE4_WR_IDX_OFF); 253 u32 rear_ri = readl(cmd_q->reg_control + AE4_RD_IDX_OFF); 254 255 if (((MAX_CMD_QLEN + front_wi - rear_ri) % MAX_CMD_QLEN) >= (MAX_CMD_QLEN - 1)) 256 return true; 257 258 return false; 259 } 260 261 static void pt_cmd_callback(void *data, int err) 262 { 263 struct pt_dma_desc *desc = data; 264 struct ae4_cmd_queue *ae4cmd_q; 265 struct dma_chan *dma_chan; 266 struct pt_dma_chan *chan; 267 struct ae4_device *ae4; 268 struct pt_device *pt; 269 int ret; 270 271 if (err == -EINPROGRESS) 272 return; 273 274 dma_chan = desc->vd.tx.chan; 275 chan = to_pt_chan(dma_chan); 276 pt = chan->pt; 277 278 if (err) 279 desc->status = DMA_ERROR; 280 281 while (true) { 282 if (pt->ver == AE4_DMA_VERSION) { 283 ae4 = container_of(pt, struct ae4_device, pt); 284 ae4cmd_q = &ae4->ae4cmd_q[chan->id]; 285 286 if (ae4cmd_q->q_cmd_count >= (CMD_Q_LEN - 1) || 287 ae4_core_queue_full(&ae4cmd_q->cmd_q)) { 288 wake_up(&ae4cmd_q->q_w); 289 290 if (wait_for_completion_timeout(&ae4cmd_q->cmp, 291 msecs_to_jiffies(AE4_TIME_OUT)) 292 == 0) { 293 dev_err(pt->dev, "TIMEOUT %d:\n", ae4cmd_q->id); 294 break; 295 } 296 297 reinit_completion(&ae4cmd_q->cmp); 298 continue; 299 } 300 } 301 302 /* Check for DMA descriptor completion */ 303 desc = pt_handle_active_desc(chan, desc); 304 305 /* Don't submit cmd if no descriptor or DMA is paused */ 306 if (!desc) 307 break; 308 309 ret = pt_dma_start_desc(desc, chan); 310 if (!ret) 311 break; 312 313 desc->status = DMA_ERROR; 314 } 315 } 316 317 static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan, 318 unsigned long flags) 319 { 320 struct pt_dma_desc *desc; 321 322 desc = kmem_cache_zalloc(chan->pt->dma_desc_cache, GFP_NOWAIT); 323 if (!desc) 324 return NULL; 325 326 vchan_tx_prep(&chan->vc, &desc->vd, flags); 327 328 desc->pt = chan->pt; 329 desc->pt->cmd_q.int_en = !!(flags & DMA_PREP_INTERRUPT); 330 desc->issued_to_hw = 0; 331 desc->status = DMA_IN_PROGRESS; 332 333 return desc; 334 } 335 336 static void pt_cmd_callback_work(void *data, int err) 337 { 338 struct dma_async_tx_descriptor *tx_desc; 339 struct pt_dma_desc *desc = data; 340 struct dma_chan *dma_chan; 341 struct virt_dma_desc *vd; 342 struct pt_dma_chan *chan; 343 unsigned long flags; 344 345 dma_chan = desc->vd.tx.chan; 346 chan = to_pt_chan(dma_chan); 347 348 if (err == -EINPROGRESS) 349 return; 350 351 tx_desc = &desc->vd.tx; 352 vd = &desc->vd; 353 354 if (err) 355 desc->status = DMA_ERROR; 356 357 spin_lock_irqsave(&chan->vc.lock, flags); 358 if (desc) { 359 if (desc->status != DMA_COMPLETE) { 360 if (desc->status != DMA_ERROR) 361 desc->status = DMA_COMPLETE; 362 363 dma_cookie_complete(tx_desc); 364 dma_descriptor_unmap(tx_desc); 365 } else { 366 tx_desc = NULL; 367 } 368 } 369 spin_unlock_irqrestore(&chan->vc.lock, flags); 370 371 if (tx_desc) { 372 dmaengine_desc_get_callback_invoke(tx_desc, NULL); 373 dma_run_dependencies(tx_desc); 374 list_del(&desc->vd.node); 375 vchan_vdesc_fini(vd); 376 } 377 } 378 379 static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, 380 dma_addr_t dst, 381 dma_addr_t src, 382 unsigned int len, 383 unsigned long flags) 384 { 385 struct pt_dma_chan *chan = to_pt_chan(dma_chan); 386 struct pt_passthru_engine *pt_engine; 387 struct pt_device *pt = chan->pt; 388 struct ae4_cmd_queue *ae4cmd_q; 389 struct pt_dma_desc *desc; 390 struct ae4_device *ae4; 391 struct pt_cmd *pt_cmd; 392 393 desc = pt_alloc_dma_desc(chan, flags); 394 if (!desc) 395 return NULL; 396 397 pt_cmd = &desc->pt_cmd; 398 pt_cmd->pt = pt; 399 pt_engine = &pt_cmd->passthru; 400 pt_cmd->engine = PT_ENGINE_PASSTHRU; 401 pt_engine->src_dma = src; 402 pt_engine->dst_dma = dst; 403 pt_engine->src_len = len; 404 pt_cmd->pt_cmd_callback = pt_cmd_callback; 405 pt_cmd->data = desc; 406 407 desc->len = len; 408 409 if (pt->ver == AE4_DMA_VERSION) { 410 pt_cmd->pt_cmd_callback = pt_cmd_callback_work; 411 ae4 = container_of(pt, struct ae4_device, pt); 412 ae4cmd_q = &ae4->ae4cmd_q[chan->id]; 413 mutex_lock(&ae4cmd_q->cmd_lock); 414 list_add_tail(&pt_cmd->entry, &ae4cmd_q->cmd); 415 mutex_unlock(&ae4cmd_q->cmd_lock); 416 } 417 418 return desc; 419 } 420 421 static struct dma_async_tx_descriptor * 422 pt_prep_dma_memcpy(struct dma_chan *dma_chan, dma_addr_t dst, 423 dma_addr_t src, size_t len, unsigned long flags) 424 { 425 struct pt_dma_desc *desc; 426 427 desc = pt_create_desc(dma_chan, dst, src, len, flags); 428 if (!desc) 429 return NULL; 430 431 return &desc->vd.tx; 432 } 433 434 static struct dma_async_tx_descriptor * 435 pt_prep_dma_interrupt(struct dma_chan *dma_chan, unsigned long flags) 436 { 437 struct pt_dma_chan *chan = to_pt_chan(dma_chan); 438 struct pt_dma_desc *desc; 439 440 desc = pt_alloc_dma_desc(chan, flags); 441 if (!desc) 442 return NULL; 443 444 return &desc->vd.tx; 445 } 446 447 static void pt_issue_pending(struct dma_chan *dma_chan) 448 { 449 struct pt_dma_chan *chan = to_pt_chan(dma_chan); 450 struct pt_dma_desc *desc; 451 struct pt_device *pt; 452 unsigned long flags; 453 bool engine_is_idle = true; 454 455 pt = chan->pt; 456 457 spin_lock_irqsave(&chan->vc.lock, flags); 458 459 desc = pt_next_dma_desc(chan); 460 if (desc && pt->ver != AE4_DMA_VERSION) 461 engine_is_idle = false; 462 463 vchan_issue_pending(&chan->vc); 464 465 desc = pt_next_dma_desc(chan); 466 467 spin_unlock_irqrestore(&chan->vc.lock, flags); 468 469 /* If there was nothing active, start processing */ 470 if (engine_is_idle && desc) 471 pt_cmd_callback(desc, 0); 472 } 473 474 static void pt_check_status_trans_ae4(struct pt_device *pt, struct pt_cmd_queue *cmd_q) 475 { 476 struct ae4_cmd_queue *ae4cmd_q = container_of(cmd_q, struct ae4_cmd_queue, cmd_q); 477 int i; 478 479 for (i = 0; i < CMD_Q_LEN; i++) 480 ae4_check_status_error(ae4cmd_q, i); 481 } 482 483 static enum dma_status 484 pt_tx_status(struct dma_chan *c, dma_cookie_t cookie, 485 struct dma_tx_state *txstate) 486 { 487 struct pt_dma_chan *chan = to_pt_chan(c); 488 struct pt_device *pt = chan->pt; 489 struct pt_cmd_queue *cmd_q; 490 491 cmd_q = pt_get_cmd_queue(pt, chan); 492 493 if (pt->ver == AE4_DMA_VERSION) 494 pt_check_status_trans_ae4(pt, cmd_q); 495 else 496 pt_check_status_trans(pt, cmd_q); 497 498 return dma_cookie_status(c, cookie, txstate); 499 } 500 501 static int pt_pause(struct dma_chan *dma_chan) 502 { 503 struct pt_dma_chan *chan = to_pt_chan(dma_chan); 504 struct pt_device *pt = chan->pt; 505 struct pt_cmd_queue *cmd_q; 506 unsigned long flags; 507 508 spin_lock_irqsave(&chan->vc.lock, flags); 509 cmd_q = pt_get_cmd_queue(pt, chan); 510 pt_stop_queue(cmd_q); 511 spin_unlock_irqrestore(&chan->vc.lock, flags); 512 513 return 0; 514 } 515 516 static int pt_resume(struct dma_chan *dma_chan) 517 { 518 struct pt_dma_chan *chan = to_pt_chan(dma_chan); 519 struct pt_dma_desc *desc = NULL; 520 struct pt_device *pt = chan->pt; 521 struct pt_cmd_queue *cmd_q; 522 unsigned long flags; 523 524 spin_lock_irqsave(&chan->vc.lock, flags); 525 cmd_q = pt_get_cmd_queue(pt, chan); 526 pt_start_queue(cmd_q); 527 desc = pt_next_dma_desc(chan); 528 spin_unlock_irqrestore(&chan->vc.lock, flags); 529 530 /* If there was something active, re-start */ 531 if (desc) 532 pt_cmd_callback(desc, 0); 533 534 return 0; 535 } 536 537 static int pt_terminate_all(struct dma_chan *dma_chan) 538 { 539 struct pt_dma_chan *chan = to_pt_chan(dma_chan); 540 struct pt_device *pt = chan->pt; 541 struct pt_cmd_queue *cmd_q; 542 unsigned long flags; 543 LIST_HEAD(head); 544 545 cmd_q = pt_get_cmd_queue(pt, chan); 546 if (pt->ver == AE4_DMA_VERSION) 547 pt_stop_queue(cmd_q); 548 else 549 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); 550 551 spin_lock_irqsave(&chan->vc.lock, flags); 552 vchan_get_all_descriptors(&chan->vc, &head); 553 spin_unlock_irqrestore(&chan->vc.lock, flags); 554 555 vchan_dma_desc_free_list(&chan->vc, &head); 556 vchan_free_chan_resources(&chan->vc); 557 558 return 0; 559 } 560 561 int pt_dmaengine_register(struct pt_device *pt) 562 { 563 struct dma_device *dma_dev = &pt->dma_dev; 564 struct ae4_cmd_queue *ae4cmd_q = NULL; 565 struct ae4_device *ae4 = NULL; 566 struct pt_dma_chan *chan; 567 char *desc_cache_name; 568 char *cmd_cache_name; 569 int ret, i; 570 571 if (pt->ver == AE4_DMA_VERSION) 572 ae4 = container_of(pt, struct ae4_device, pt); 573 574 if (ae4) 575 pt->pt_dma_chan = devm_kcalloc(pt->dev, ae4->cmd_q_count, 576 sizeof(*pt->pt_dma_chan), GFP_KERNEL); 577 else 578 pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan), 579 GFP_KERNEL); 580 581 if (!pt->pt_dma_chan) 582 return -ENOMEM; 583 584 cmd_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL, 585 "%s-dmaengine-cmd-cache", 586 dev_name(pt->dev)); 587 if (!cmd_cache_name) 588 return -ENOMEM; 589 590 desc_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL, 591 "%s-dmaengine-desc-cache", 592 dev_name(pt->dev)); 593 if (!desc_cache_name) { 594 ret = -ENOMEM; 595 goto err_cache; 596 } 597 598 pt->dma_desc_cache = kmem_cache_create(desc_cache_name, 599 sizeof(struct pt_dma_desc), 0, 600 SLAB_HWCACHE_ALIGN, NULL); 601 if (!pt->dma_desc_cache) { 602 ret = -ENOMEM; 603 goto err_cache; 604 } 605 606 dma_dev->dev = pt->dev; 607 dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; 608 dma_dev->dst_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; 609 dma_dev->directions = DMA_MEM_TO_MEM; 610 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 611 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 612 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); 613 614 /* 615 * PTDMA is intended to be used with the AMD NTB devices, hence 616 * marking it as DMA_PRIVATE. 617 */ 618 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); 619 620 INIT_LIST_HEAD(&dma_dev->channels); 621 622 /* Set base and prep routines */ 623 dma_dev->device_free_chan_resources = pt_free_chan_resources; 624 dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy; 625 dma_dev->device_prep_dma_interrupt = pt_prep_dma_interrupt; 626 dma_dev->device_issue_pending = pt_issue_pending; 627 dma_dev->device_tx_status = pt_tx_status; 628 dma_dev->device_pause = pt_pause; 629 dma_dev->device_resume = pt_resume; 630 dma_dev->device_terminate_all = pt_terminate_all; 631 dma_dev->device_synchronize = pt_synchronize; 632 633 if (ae4) { 634 for (i = 0; i < ae4->cmd_q_count; i++) { 635 chan = pt->pt_dma_chan + i; 636 ae4cmd_q = &ae4->ae4cmd_q[i]; 637 chan->id = ae4cmd_q->id; 638 chan->pt = pt; 639 chan->vc.desc_free = pt_do_cleanup; 640 vchan_init(&chan->vc, dma_dev); 641 } 642 } else { 643 chan = pt->pt_dma_chan; 644 chan->pt = pt; 645 chan->vc.desc_free = pt_do_cleanup; 646 vchan_init(&chan->vc, dma_dev); 647 } 648 649 ret = dma_async_device_register(dma_dev); 650 if (ret) 651 goto err_reg; 652 653 return 0; 654 655 err_reg: 656 kmem_cache_destroy(pt->dma_desc_cache); 657 658 err_cache: 659 kmem_cache_destroy(pt->dma_cmd_cache); 660 661 return ret; 662 } 663 EXPORT_SYMBOL_GPL(pt_dmaengine_register); 664 665 void pt_dmaengine_unregister(struct pt_device *pt) 666 { 667 struct dma_device *dma_dev = &pt->dma_dev; 668 669 dma_async_device_unregister(dma_dev); 670 671 kmem_cache_destroy(pt->dma_desc_cache); 672 kmem_cache_destroy(pt->dma_cmd_cache); 673 } 674