1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #ifndef _IDXD_H_ 4 #define _IDXD_H_ 5 6 #include <linux/sbitmap.h> 7 #include <linux/dmaengine.h> 8 #include <linux/percpu-rwsem.h> 9 #include <linux/wait.h> 10 #include <linux/cdev.h> 11 #include <linux/idr.h> 12 #include <linux/pci.h> 13 #include <linux/bitmap.h> 14 #include <linux/perf_event.h> 15 #include <linux/iommu.h> 16 #include <linux/crypto.h> 17 #include <uapi/linux/idxd.h> 18 #include "registers.h" 19 20 #define IDXD_DRIVER_VERSION "1.00" 21 22 extern bool tc_override; 23 24 struct idxd_wq; 25 struct idxd_dev; 26 27 enum idxd_dev_type { 28 IDXD_DEV_NONE = -1, 29 IDXD_DEV_DSA = 0, 30 IDXD_DEV_IAX, 31 IDXD_DEV_WQ, 32 IDXD_DEV_GROUP, 33 IDXD_DEV_ENGINE, 34 IDXD_DEV_CDEV, 35 IDXD_DEV_CDEV_FILE, 36 IDXD_DEV_MAX_TYPE, 37 }; 38 39 struct idxd_dev { 40 struct device conf_dev; 41 enum idxd_dev_type type; 42 }; 43 44 #define IDXD_REG_TIMEOUT 50 45 #define IDXD_DRAIN_TIMEOUT 5000 46 47 enum idxd_type { 48 IDXD_TYPE_UNKNOWN = -1, 49 IDXD_TYPE_DSA = 0, 50 IDXD_TYPE_IAX, 51 IDXD_TYPE_MAX, 52 }; 53 54 #define IDXD_NAME_SIZE 128 55 #define IDXD_PMU_EVENT_MAX 64 56 57 #define IDXD_ENQCMDS_RETRIES 32 58 #define IDXD_ENQCMDS_MAX_RETRIES 64 59 60 enum idxd_complete_type { 61 IDXD_COMPLETE_NORMAL = 0, 62 IDXD_COMPLETE_ABORT, 63 IDXD_COMPLETE_DEV_FAIL, 64 }; 65 66 struct idxd_desc; 67 68 struct idxd_device_driver { 69 const char *name; 70 enum idxd_dev_type *type; 71 int (*probe)(struct idxd_dev *idxd_dev); 72 void (*remove)(struct idxd_dev *idxd_dev); 73 void (*desc_complete)(struct idxd_desc *desc, 74 enum idxd_complete_type comp_type, 75 bool free_desc, 76 void *ctx, u32 *status); 77 struct device_driver drv; 78 }; 79 80 extern struct idxd_device_driver dsa_drv; 81 extern struct idxd_device_driver idxd_drv; 82 extern struct idxd_device_driver idxd_dmaengine_drv; 83 extern struct idxd_device_driver idxd_user_drv; 84 85 #define INVALID_INT_HANDLE -1 86 struct idxd_irq_entry { 87 int id; 88 int vector; 89 struct llist_head pending_llist; 90 struct list_head work_list; 91 /* 92 * Lock to protect access between irq thread process descriptor 93 * and irq thread processing error descriptor. 94 */ 95 spinlock_t list_lock; 96 int int_handle; 97 ioasid_t pasid; 98 }; 99 100 struct idxd_group { 101 struct idxd_dev idxd_dev; 102 struct idxd_device *idxd; 103 struct grpcfg grpcfg; 104 int id; 105 int num_engines; 106 int num_wqs; 107 bool use_rdbuf_limit; 108 u8 rdbufs_allowed; 109 u8 rdbufs_reserved; 110 int tc_a; 111 int tc_b; 112 int desc_progress_limit; 113 int batch_progress_limit; 114 }; 115 116 struct idxd_pmu { 117 struct idxd_device *idxd; 118 119 struct perf_event *event_list[IDXD_PMU_EVENT_MAX]; 120 int n_events; 121 122 DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX); 123 124 struct pmu pmu; 125 char name[IDXD_NAME_SIZE]; 126 127 int n_counters; 128 int counter_width; 129 int n_event_categories; 130 131 bool per_counter_caps_supported; 132 unsigned long supported_event_categories; 133 134 unsigned long supported_filters; 135 int n_filters; 136 }; 137 138 #define IDXD_MAX_PRIORITY 0xf 139 140 enum { 141 COUNTER_FAULTS = 0, 142 COUNTER_FAULT_FAILS, 143 COUNTER_MAX 144 }; 145 146 enum idxd_wq_state { 147 IDXD_WQ_DISABLED = 0, 148 IDXD_WQ_ENABLED, 149 }; 150 151 enum idxd_wq_flag { 152 WQ_FLAG_DEDICATED = 0, 153 WQ_FLAG_BLOCK_ON_FAULT, 154 WQ_FLAG_ATS_DISABLE, 155 WQ_FLAG_PRS_DISABLE, 156 }; 157 158 enum idxd_wq_type { 159 IDXD_WQT_NONE = 0, 160 IDXD_WQT_KERNEL, 161 IDXD_WQT_USER, 162 }; 163 164 struct idxd_cdev { 165 struct idxd_wq *wq; 166 struct cdev cdev; 167 struct idxd_dev idxd_dev; 168 int minor; 169 }; 170 171 #define DRIVER_NAME_SIZE 128 172 173 #define WQ_NAME_SIZE 1024 174 #define WQ_TYPE_SIZE 10 175 176 #define WQ_DEFAULT_QUEUE_DEPTH 16 177 #define WQ_DEFAULT_MAX_XFER SZ_2M 178 #define WQ_DEFAULT_MAX_BATCH 32 179 180 enum idxd_op_type { 181 IDXD_OP_BLOCK = 0, 182 IDXD_OP_NONBLOCK = 1, 183 }; 184 185 struct idxd_dma_chan { 186 struct dma_chan chan; 187 struct idxd_wq *wq; 188 }; 189 190 struct idxd_wq { 191 void __iomem *portal; 192 u32 portal_offset; 193 unsigned int enqcmds_retries; 194 struct percpu_ref wq_active; 195 struct completion wq_dead; 196 struct completion wq_resurrect; 197 struct idxd_dev idxd_dev; 198 struct idxd_cdev *idxd_cdev; 199 struct wait_queue_head err_queue; 200 struct workqueue_struct *wq; 201 struct idxd_device *idxd; 202 int id; 203 struct idxd_irq_entry ie; 204 enum idxd_wq_type type; 205 struct idxd_group *group; 206 int client_count; 207 struct mutex wq_lock; /* mutex for workqueue */ 208 u32 size; 209 u32 threshold; 210 u32 priority; 211 enum idxd_wq_state state; 212 unsigned long flags; 213 union wqcfg *wqcfg; 214 unsigned long *opcap_bmap; 215 216 struct dsa_hw_desc **hw_descs; 217 int num_descs; 218 union { 219 struct dsa_completion_record *compls; 220 struct iax_completion_record *iax_compls; 221 }; 222 dma_addr_t compls_addr; 223 int compls_size; 224 struct idxd_desc **descs; 225 struct sbitmap_queue sbq; 226 struct idxd_dma_chan *idxd_chan; 227 char name[WQ_NAME_SIZE + 1]; 228 u64 max_xfer_bytes; 229 u32 max_batch_size; 230 231 /* Lock to protect upasid_xa access. */ 232 struct mutex uc_lock; 233 struct xarray upasid_xa; 234 235 char driver_name[DRIVER_NAME_SIZE + 1]; 236 }; 237 238 struct idxd_engine { 239 struct idxd_dev idxd_dev; 240 int id; 241 struct idxd_group *group; 242 struct idxd_device *idxd; 243 }; 244 245 /* shadow registers */ 246 struct idxd_hw { 247 u32 version; 248 union gen_cap_reg gen_cap; 249 union wq_cap_reg wq_cap; 250 union group_cap_reg group_cap; 251 union engine_cap_reg engine_cap; 252 struct opcap opcap; 253 u32 cmd_cap; 254 union iaa_cap_reg iaa_cap; 255 }; 256 257 enum idxd_device_state { 258 IDXD_DEV_HALTED = -1, 259 IDXD_DEV_DISABLED = 0, 260 IDXD_DEV_ENABLED, 261 }; 262 263 enum idxd_device_flag { 264 IDXD_FLAG_CONFIGURABLE = 0, 265 IDXD_FLAG_CMD_RUNNING, 266 IDXD_FLAG_PASID_ENABLED, 267 IDXD_FLAG_USER_PASID_ENABLED, 268 }; 269 270 struct idxd_dma_dev { 271 struct idxd_device *idxd; 272 struct dma_device dma; 273 }; 274 275 typedef int (*load_device_defaults_fn_t) (struct idxd_device *idxd); 276 277 struct idxd_driver_data { 278 const char *name_prefix; 279 enum idxd_type type; 280 const struct device_type *dev_type; 281 int compl_size; 282 int align; 283 int evl_cr_off; 284 int cr_status_off; 285 int cr_result_off; 286 bool user_submission_safe; 287 load_device_defaults_fn_t load_device_defaults; 288 }; 289 290 struct idxd_evl { 291 /* Lock to protect event log access. */ 292 struct mutex lock; 293 void *log; 294 dma_addr_t dma; 295 /* Total size of event log = number of entries * entry size. */ 296 unsigned int log_size; 297 /* The number of entries in the event log. */ 298 u16 size; 299 unsigned long *bmap; 300 bool batch_fail[IDXD_MAX_BATCH_IDENT]; 301 }; 302 303 struct idxd_evl_fault { 304 struct work_struct work; 305 struct idxd_wq *wq; 306 u8 status; 307 308 /* make this last member always */ 309 struct __evl_entry entry[]; 310 }; 311 312 struct idxd_device { 313 struct idxd_dev idxd_dev; 314 struct idxd_driver_data *data; 315 struct list_head list; 316 struct idxd_hw hw; 317 enum idxd_device_state state; 318 unsigned long flags; 319 int id; 320 int major; 321 u32 cmd_status; 322 struct idxd_irq_entry ie; /* misc irq, msix 0 */ 323 324 struct pci_dev *pdev; 325 void __iomem *reg_base; 326 327 spinlock_t dev_lock; /* spinlock for device */ 328 spinlock_t cmd_lock; /* spinlock for device commands */ 329 struct completion *cmd_done; 330 struct idxd_group **groups; 331 struct idxd_wq **wqs; 332 struct idxd_engine **engines; 333 334 struct iommu_sva *sva; 335 unsigned int pasid; 336 337 int num_groups; 338 int irq_cnt; 339 bool request_int_handles; 340 341 u32 msix_perm_offset; 342 u32 wqcfg_offset; 343 u32 grpcfg_offset; 344 u32 perfmon_offset; 345 346 u64 max_xfer_bytes; 347 u32 max_batch_size; 348 int max_groups; 349 int max_engines; 350 int max_rdbufs; 351 int max_wqs; 352 int max_wq_size; 353 int rdbuf_limit; 354 int nr_rdbufs; /* non-reserved read buffers */ 355 unsigned int wqcfg_size; 356 unsigned long *wq_enable_map; 357 358 union sw_err_reg sw_err; 359 wait_queue_head_t cmd_waitq; 360 361 struct idxd_dma_dev *idxd_dma; 362 struct workqueue_struct *wq; 363 struct work_struct work; 364 365 struct idxd_pmu *idxd_pmu; 366 367 unsigned long *opcap_bmap; 368 struct idxd_evl *evl; 369 struct kmem_cache *evl_cache; 370 371 struct dentry *dbgfs_dir; 372 struct dentry *dbgfs_evl_file; 373 374 bool user_submission_safe; 375 376 struct idxd_saved_states *idxd_saved; 377 }; 378 379 struct idxd_saved_states { 380 struct idxd_device saved_idxd; 381 struct idxd_evl saved_evl; 382 struct idxd_engine **saved_engines; 383 struct idxd_wq **saved_wqs; 384 struct idxd_group **saved_groups; 385 unsigned long *saved_wq_enable_map; 386 }; 387 388 static inline unsigned int evl_ent_size(struct idxd_device *idxd) 389 { 390 return idxd->hw.gen_cap.evl_support ? 391 (32 * (1 << idxd->hw.gen_cap.evl_support)) : 0; 392 } 393 394 static inline unsigned int evl_size(struct idxd_device *idxd) 395 { 396 return idxd->evl->size * evl_ent_size(idxd); 397 } 398 399 struct crypto_ctx { 400 struct acomp_req *req; 401 struct crypto_tfm *tfm; 402 dma_addr_t src_addr; 403 dma_addr_t dst_addr; 404 bool compress; 405 }; 406 407 /* IDXD software descriptor */ 408 struct idxd_desc { 409 union { 410 struct dsa_hw_desc *hw; 411 struct iax_hw_desc *iax_hw; 412 }; 413 dma_addr_t desc_dma; 414 union { 415 struct dsa_completion_record *completion; 416 struct iax_completion_record *iax_completion; 417 }; 418 dma_addr_t compl_dma; 419 union { 420 struct dma_async_tx_descriptor txd; 421 struct crypto_ctx crypto; 422 }; 423 struct llist_node llnode; 424 struct list_head list; 425 int id; 426 int cpu; 427 struct idxd_wq *wq; 428 }; 429 430 /* 431 * This is software defined error for the completion status. We overload the error code 432 * that will never appear in completion status and only SWERR register. 433 */ 434 enum idxd_completion_status { 435 IDXD_COMP_DESC_ABORT = 0xff, 436 }; 437 438 #define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev 439 #define wq_confdev(wq) &wq->idxd_dev.conf_dev 440 #define engine_confdev(engine) &engine->idxd_dev.conf_dev 441 #define group_confdev(group) &group->idxd_dev.conf_dev 442 #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev 443 #define user_ctx_dev(ctx) (&(ctx)->idxd_dev.conf_dev) 444 445 #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev) 446 #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev) 447 #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev) 448 449 static inline struct idxd_device_driver *wq_to_idxd_drv(struct idxd_wq *wq) 450 { 451 struct device *dev = wq_confdev(wq); 452 struct idxd_device_driver *idxd_drv = 453 container_of(dev->driver, struct idxd_device_driver, drv); 454 455 return idxd_drv; 456 } 457 458 static inline struct idxd_device *confdev_to_idxd(struct device *dev) 459 { 460 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 461 462 return idxd_dev_to_idxd(idxd_dev); 463 } 464 465 static inline struct idxd_wq *confdev_to_wq(struct device *dev) 466 { 467 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 468 469 return idxd_dev_to_wq(idxd_dev); 470 } 471 472 static inline struct idxd_engine *confdev_to_engine(struct device *dev) 473 { 474 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 475 476 return container_of(idxd_dev, struct idxd_engine, idxd_dev); 477 } 478 479 static inline struct idxd_group *confdev_to_group(struct device *dev) 480 { 481 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 482 483 return container_of(idxd_dev, struct idxd_group, idxd_dev); 484 } 485 486 static inline struct idxd_cdev *dev_to_cdev(struct device *dev) 487 { 488 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 489 490 return container_of(idxd_dev, struct idxd_cdev, idxd_dev); 491 } 492 493 static inline void idxd_dev_set_type(struct idxd_dev *idev, int type) 494 { 495 if (type >= IDXD_DEV_MAX_TYPE) { 496 idev->type = IDXD_DEV_NONE; 497 return; 498 } 499 500 idev->type = type; 501 } 502 503 static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx) 504 { 505 return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie; 506 } 507 508 static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie) 509 { 510 return container_of(ie, struct idxd_wq, ie); 511 } 512 513 static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie) 514 { 515 return container_of(ie, struct idxd_device, ie); 516 } 517 518 static inline void idxd_set_user_intr(struct idxd_device *idxd, bool enable) 519 { 520 union gencfg_reg reg; 521 522 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); 523 reg.user_int_en = enable; 524 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); 525 } 526 527 extern const struct bus_type dsa_bus_type; 528 529 extern bool support_enqcmd; 530 extern struct ida idxd_ida; 531 extern const struct device_type dsa_device_type; 532 extern const struct device_type iax_device_type; 533 extern const struct device_type idxd_wq_device_type; 534 extern const struct device_type idxd_engine_device_type; 535 extern const struct device_type idxd_group_device_type; 536 537 static inline bool is_dsa_dev(struct idxd_dev *idxd_dev) 538 { 539 return idxd_dev->type == IDXD_DEV_DSA; 540 } 541 542 static inline bool is_iax_dev(struct idxd_dev *idxd_dev) 543 { 544 return idxd_dev->type == IDXD_DEV_IAX; 545 } 546 547 static inline bool is_idxd_dev(struct idxd_dev *idxd_dev) 548 { 549 return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev); 550 } 551 552 static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev) 553 { 554 return idxd_dev->type == IDXD_DEV_WQ; 555 } 556 557 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) 558 { 559 if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0) 560 return true; 561 return false; 562 } 563 564 static inline bool is_idxd_wq_user(struct idxd_wq *wq) 565 { 566 return wq->type == IDXD_WQT_USER; 567 } 568 569 static inline bool is_idxd_wq_kernel(struct idxd_wq *wq) 570 { 571 return wq->type == IDXD_WQT_KERNEL; 572 } 573 574 static inline bool wq_dedicated(struct idxd_wq *wq) 575 { 576 return test_bit(WQ_FLAG_DEDICATED, &wq->flags); 577 } 578 579 static inline bool wq_shared(struct idxd_wq *wq) 580 { 581 return !test_bit(WQ_FLAG_DEDICATED, &wq->flags); 582 } 583 584 static inline bool device_pasid_enabled(struct idxd_device *idxd) 585 { 586 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 587 } 588 589 static inline bool device_user_pasid_enabled(struct idxd_device *idxd) 590 { 591 return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 592 } 593 594 static inline bool wq_pasid_enabled(struct idxd_wq *wq) 595 { 596 return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) || 597 (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd)); 598 } 599 600 static inline bool wq_shared_supported(struct idxd_wq *wq) 601 { 602 return (support_enqcmd && wq_pasid_enabled(wq)); 603 } 604 605 enum idxd_portal_prot { 606 IDXD_PORTAL_UNLIMITED = 0, 607 IDXD_PORTAL_LIMITED, 608 }; 609 610 enum idxd_interrupt_type { 611 IDXD_IRQ_MSIX = 0, 612 IDXD_IRQ_IMS, 613 }; 614 615 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot) 616 { 617 return prot * 0x1000; 618 } 619 620 static inline int idxd_get_wq_portal_full_offset(int wq_id, 621 enum idxd_portal_prot prot) 622 { 623 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); 624 } 625 626 #define IDXD_PORTAL_MASK (PAGE_SIZE - 1) 627 628 /* 629 * Even though this function can be accessed by multiple threads, it is safe to use. 630 * At worst the address gets used more than once before it gets incremented. We don't 631 * hit a threshold until iops becomes many million times a second. So the occasional 632 * reuse of the same address is tolerable compare to using an atomic variable. This is 633 * safe on a system that has atomic load/store for 32bit integers. Given that this is an 634 * Intel iEP device, that should not be a problem. 635 */ 636 static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq) 637 { 638 int ofs = wq->portal_offset; 639 640 wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK; 641 return wq->portal + ofs; 642 } 643 644 static inline void idxd_wq_get(struct idxd_wq *wq) 645 { 646 wq->client_count++; 647 } 648 649 static inline void idxd_wq_put(struct idxd_wq *wq) 650 { 651 wq->client_count--; 652 } 653 654 static inline int idxd_wq_refcount(struct idxd_wq *wq) 655 { 656 return wq->client_count; 657 }; 658 659 static inline void idxd_wq_set_private(struct idxd_wq *wq, void *private) 660 { 661 dev_set_drvdata(wq_confdev(wq), private); 662 } 663 664 static inline void *idxd_wq_get_private(struct idxd_wq *wq) 665 { 666 return dev_get_drvdata(wq_confdev(wq)); 667 } 668 669 /* 670 * Intel IAA does not support batch processing. 671 * The max batch size of device, max batch size of wq and 672 * max batch shift of wqcfg should be always 0 on IAA. 673 */ 674 static inline void idxd_set_max_batch_size(int idxd_type, struct idxd_device *idxd, 675 u32 max_batch_size) 676 { 677 if (idxd_type == IDXD_TYPE_IAX) 678 idxd->max_batch_size = 0; 679 else 680 idxd->max_batch_size = max_batch_size; 681 } 682 683 static inline void idxd_wq_set_max_batch_size(int idxd_type, struct idxd_wq *wq, 684 u32 max_batch_size) 685 { 686 if (idxd_type == IDXD_TYPE_IAX) 687 wq->max_batch_size = 0; 688 else 689 wq->max_batch_size = max_batch_size; 690 } 691 692 static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wqcfg, 693 u32 max_batch_shift) 694 { 695 if (idxd_type == IDXD_TYPE_IAX) 696 wqcfg->max_batch_shift = 0; 697 else 698 wqcfg->max_batch_shift = max_batch_shift; 699 } 700 701 static inline int idxd_wq_driver_name_match(struct idxd_wq *wq, struct device *dev) 702 { 703 return (strncmp(wq->driver_name, dev->driver->name, strlen(dev->driver->name)) == 0); 704 } 705 706 #define MODULE_ALIAS_IDXD_DEVICE(type) MODULE_ALIAS("idxd:t" __stringify(type) "*") 707 #define IDXD_DEVICES_MODALIAS_FMT "idxd:t%d" 708 709 int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, 710 struct module *module, const char *mod_name); 711 #define idxd_driver_register(driver) \ 712 __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) 713 714 void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); 715 716 #define module_idxd_driver(__idxd_driver) \ 717 module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister) 718 719 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); 720 void idxd_dma_complete_txd(struct idxd_desc *desc, 721 enum idxd_complete_type comp_type, 722 bool free_desc, void *ctx, u32 *status); 723 724 static inline void idxd_desc_complete(struct idxd_desc *desc, 725 enum idxd_complete_type comp_type, 726 bool free_desc) 727 { 728 struct idxd_device_driver *drv; 729 u32 status; 730 731 drv = wq_to_idxd_drv(desc->wq); 732 if (drv->desc_complete) 733 drv->desc_complete(desc, comp_type, free_desc, 734 &desc->txd, &status); 735 } 736 737 int idxd_register_devices(struct idxd_device *idxd); 738 void idxd_unregister_devices(struct idxd_device *idxd); 739 void idxd_wqs_quiesce(struct idxd_device *idxd); 740 bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc); 741 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count); 742 int idxd_load_iaa_device_defaults(struct idxd_device *idxd); 743 744 /* device interrupt control */ 745 irqreturn_t idxd_misc_thread(int vec, void *data); 746 irqreturn_t idxd_wq_thread(int irq, void *data); 747 void idxd_mask_error_interrupts(struct idxd_device *idxd); 748 void idxd_unmask_error_interrupts(struct idxd_device *idxd); 749 750 /* device control */ 751 int idxd_device_drv_probe(struct idxd_dev *idxd_dev); 752 int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev, 753 const struct pci_device_id *id); 754 void idxd_device_drv_remove(struct idxd_dev *idxd_dev); 755 int idxd_drv_enable_wq(struct idxd_wq *wq); 756 void idxd_drv_disable_wq(struct idxd_wq *wq); 757 int idxd_device_init_reset(struct idxd_device *idxd); 758 int idxd_device_enable(struct idxd_device *idxd); 759 int idxd_device_disable(struct idxd_device *idxd); 760 void idxd_device_reset(struct idxd_device *idxd); 761 void idxd_device_clear_state(struct idxd_device *idxd); 762 int idxd_device_config(struct idxd_device *idxd); 763 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); 764 int idxd_device_load_config(struct idxd_device *idxd); 765 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, 766 enum idxd_interrupt_type irq_type); 767 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, 768 enum idxd_interrupt_type irq_type); 769 770 /* work queue control */ 771 void idxd_wqs_unmap_portal(struct idxd_device *idxd); 772 int idxd_wq_alloc_resources(struct idxd_wq *wq); 773 void idxd_wq_free_resources(struct idxd_wq *wq); 774 int idxd_wq_enable(struct idxd_wq *wq); 775 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config); 776 void idxd_wq_drain(struct idxd_wq *wq); 777 void idxd_wq_reset(struct idxd_wq *wq); 778 int idxd_wq_map_portal(struct idxd_wq *wq); 779 void idxd_wq_unmap_portal(struct idxd_wq *wq); 780 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid); 781 int idxd_wq_disable_pasid(struct idxd_wq *wq); 782 void __idxd_wq_quiesce(struct idxd_wq *wq); 783 void idxd_wq_quiesce(struct idxd_wq *wq); 784 int idxd_wq_init_percpu_ref(struct idxd_wq *wq); 785 void idxd_wq_free_irq(struct idxd_wq *wq); 786 int idxd_wq_request_irq(struct idxd_wq *wq); 787 788 /* submission */ 789 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); 790 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype); 791 int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc); 792 793 /* dmaengine */ 794 int idxd_register_dma_device(struct idxd_device *idxd); 795 void idxd_unregister_dma_device(struct idxd_device *idxd); 796 797 /* cdev */ 798 int idxd_cdev_register(void); 799 void idxd_cdev_remove(void); 800 int idxd_cdev_get_major(struct idxd_device *idxd); 801 int idxd_wq_add_cdev(struct idxd_wq *wq); 802 void idxd_wq_del_cdev(struct idxd_wq *wq); 803 int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr, 804 void *buf, int len); 805 void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index); 806 807 /* perfmon */ 808 #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON) 809 int perfmon_pmu_init(struct idxd_device *idxd); 810 void perfmon_pmu_remove(struct idxd_device *idxd); 811 void perfmon_counter_overflow(struct idxd_device *idxd); 812 #else 813 static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; } 814 static inline void perfmon_pmu_remove(struct idxd_device *idxd) {} 815 static inline void perfmon_counter_overflow(struct idxd_device *idxd) {} 816 #endif 817 818 /* debugfs */ 819 int idxd_device_init_debugfs(struct idxd_device *idxd); 820 void idxd_device_remove_debugfs(struct idxd_device *idxd); 821 int idxd_init_debugfs(void); 822 void idxd_remove_debugfs(void); 823 824 #endif 825