1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #include <linux/init.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/slab.h>
7 #include <linux/pci.h>
8 #include <linux/interrupt.h>
9 #include <linux/delay.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/workqueue.h>
12 #include <linux/fs.h>
13 #include <linux/io-64-nonatomic-lo-hi.h>
14 #include <linux/device.h>
15 #include <linux/idr.h>
16 #include <linux/iommu.h>
17 #include <uapi/linux/idxd.h>
18 #include <linux/dmaengine.h>
19 #include "../dmaengine.h"
20 #include "registers.h"
21 #include "idxd.h"
22 #include "perfmon.h"
23
24 MODULE_VERSION(IDXD_DRIVER_VERSION);
25 MODULE_DESCRIPTION("Intel Data Streaming Accelerator and In-Memory Analytics Accelerator common driver");
26 MODULE_LICENSE("GPL v2");
27 MODULE_AUTHOR("Intel Corporation");
28 MODULE_IMPORT_NS("IDXD");
29
30 static bool sva = true;
31 module_param(sva, bool, 0644);
32 MODULE_PARM_DESC(sva, "Toggle SVA support on/off");
33
34 bool tc_override;
35 module_param(tc_override, bool, 0644);
36 MODULE_PARM_DESC(tc_override, "Override traffic class defaults");
37
38 #define DRV_NAME "idxd"
39
40 bool support_enqcmd;
41 DEFINE_IDA(idxd_ida);
42
43 static struct idxd_driver_data idxd_driver_data[] = {
44 [IDXD_TYPE_DSA] = {
45 .name_prefix = "dsa",
46 .type = IDXD_TYPE_DSA,
47 .compl_size = sizeof(struct dsa_completion_record),
48 .align = 32,
49 .dev_type = &dsa_device_type,
50 .evl_cr_off = offsetof(struct dsa_evl_entry, cr),
51 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */
52 .cr_status_off = offsetof(struct dsa_completion_record, status),
53 .cr_result_off = offsetof(struct dsa_completion_record, result),
54 },
55 [IDXD_TYPE_IAX] = {
56 .name_prefix = "iax",
57 .type = IDXD_TYPE_IAX,
58 .compl_size = sizeof(struct iax_completion_record),
59 .align = 64,
60 .dev_type = &iax_device_type,
61 .evl_cr_off = offsetof(struct iax_evl_entry, cr),
62 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */
63 .cr_status_off = offsetof(struct iax_completion_record, status),
64 .cr_result_off = offsetof(struct iax_completion_record, error_code),
65 .load_device_defaults = idxd_load_iaa_device_defaults,
66 },
67 };
68
69 static struct pci_device_id idxd_pci_tbl[] = {
70 /* DSA ver 1.0 platforms */
71 { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) },
72 /* DSA on GNR-D platforms */
73 { PCI_DEVICE_DATA(INTEL, DSA_GNRD, &idxd_driver_data[IDXD_TYPE_DSA]) },
74 /* DSA on DMR platforms */
75 { PCI_DEVICE_DATA(INTEL, DSA_DMR, &idxd_driver_data[IDXD_TYPE_DSA]) },
76
77 /* IAX ver 1.0 platforms */
78 { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) },
79 /* IAA on DMR platforms */
80 { PCI_DEVICE_DATA(INTEL, IAA_DMR, &idxd_driver_data[IDXD_TYPE_IAX]) },
81 /* IAA PTL platforms */
82 { PCI_DEVICE_DATA(INTEL, IAA_PTL, &idxd_driver_data[IDXD_TYPE_IAX]) },
83 { 0, }
84 };
85 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);
86
idxd_setup_interrupts(struct idxd_device * idxd)87 static int idxd_setup_interrupts(struct idxd_device *idxd)
88 {
89 struct pci_dev *pdev = idxd->pdev;
90 struct device *dev = &pdev->dev;
91 struct idxd_irq_entry *ie;
92 int i, msixcnt;
93 int rc = 0;
94
95 msixcnt = pci_msix_vec_count(pdev);
96 if (msixcnt < 0) {
97 dev_err(dev, "Not MSI-X interrupt capable.\n");
98 return -ENOSPC;
99 }
100 idxd->irq_cnt = msixcnt;
101
102 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX);
103 if (rc != msixcnt) {
104 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc);
105 return -ENOSPC;
106 }
107 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt);
108
109
110 ie = idxd_get_ie(idxd, 0);
111 ie->vector = pci_irq_vector(pdev, 0);
112 rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie);
113 if (rc < 0) {
114 dev_err(dev, "Failed to allocate misc interrupt.\n");
115 goto err_misc_irq;
116 }
117 dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector);
118
119 for (i = 0; i < idxd->max_wqs; i++) {
120 int msix_idx = i + 1;
121
122 ie = idxd_get_ie(idxd, msix_idx);
123 ie->id = msix_idx;
124 ie->int_handle = INVALID_INT_HANDLE;
125 ie->pasid = IOMMU_PASID_INVALID;
126
127 spin_lock_init(&ie->list_lock);
128 init_llist_head(&ie->pending_llist);
129 INIT_LIST_HEAD(&ie->work_list);
130 }
131
132 idxd_unmask_error_interrupts(idxd);
133 return 0;
134
135 err_misc_irq:
136 idxd_mask_error_interrupts(idxd);
137 pci_free_irq_vectors(pdev);
138 dev_err(dev, "No usable interrupts\n");
139 return rc;
140 }
141
idxd_cleanup_interrupts(struct idxd_device * idxd)142 static void idxd_cleanup_interrupts(struct idxd_device *idxd)
143 {
144 struct pci_dev *pdev = idxd->pdev;
145 struct idxd_irq_entry *ie;
146 int msixcnt;
147
148 msixcnt = pci_msix_vec_count(pdev);
149 if (msixcnt <= 0)
150 return;
151
152 ie = idxd_get_ie(idxd, 0);
153 idxd_mask_error_interrupts(idxd);
154 free_irq(ie->vector, ie);
155 pci_free_irq_vectors(pdev);
156 }
157
idxd_clean_wqs(struct idxd_device * idxd)158 static void idxd_clean_wqs(struct idxd_device *idxd)
159 {
160 struct idxd_wq *wq;
161 struct device *conf_dev;
162 int i;
163
164 for (i = 0; i < idxd->max_wqs; i++) {
165 wq = idxd->wqs[i];
166 if (idxd->hw.wq_cap.op_config)
167 bitmap_free(wq->opcap_bmap);
168 kfree(wq->wqcfg);
169 conf_dev = wq_confdev(wq);
170 put_device(conf_dev);
171 kfree(wq);
172 }
173 bitmap_free(idxd->wq_enable_map);
174 kfree(idxd->wqs);
175 }
176
idxd_setup_wqs(struct idxd_device * idxd)177 static int idxd_setup_wqs(struct idxd_device *idxd)
178 {
179 struct device *dev = &idxd->pdev->dev;
180 struct idxd_wq *wq;
181 struct device *conf_dev;
182 int i, rc;
183
184 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *),
185 GFP_KERNEL, dev_to_node(dev));
186 if (!idxd->wqs)
187 return -ENOMEM;
188
189 idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev));
190 if (!idxd->wq_enable_map) {
191 rc = -ENOMEM;
192 goto err_bitmap;
193 }
194
195 for (i = 0; i < idxd->max_wqs; i++) {
196 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev));
197 if (!wq) {
198 rc = -ENOMEM;
199 goto err;
200 }
201
202 idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ);
203 conf_dev = wq_confdev(wq);
204 wq->id = i;
205 wq->idxd = idxd;
206 device_initialize(wq_confdev(wq));
207 conf_dev->parent = idxd_confdev(idxd);
208 conf_dev->bus = &dsa_bus_type;
209 conf_dev->type = &idxd_wq_device_type;
210 rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id);
211 if (rc < 0)
212 goto err;
213
214 mutex_init(&wq->wq_lock);
215 init_waitqueue_head(&wq->err_queue);
216 init_completion(&wq->wq_dead);
217 init_completion(&wq->wq_resurrect);
218 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER;
219 idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH);
220 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES;
221 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev));
222 if (!wq->wqcfg) {
223 rc = -ENOMEM;
224 goto err;
225 }
226
227 if (idxd->hw.wq_cap.op_config) {
228 wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL);
229 if (!wq->opcap_bmap) {
230 rc = -ENOMEM;
231 goto err_opcap_bmap;
232 }
233 bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS);
234 }
235 mutex_init(&wq->uc_lock);
236 xa_init(&wq->upasid_xa);
237 idxd->wqs[i] = wq;
238 }
239
240 return 0;
241
242 err_opcap_bmap:
243 kfree(wq->wqcfg);
244
245 err:
246 put_device(conf_dev);
247 kfree(wq);
248
249 while (--i >= 0) {
250 wq = idxd->wqs[i];
251 if (idxd->hw.wq_cap.op_config)
252 bitmap_free(wq->opcap_bmap);
253 kfree(wq->wqcfg);
254 conf_dev = wq_confdev(wq);
255 put_device(conf_dev);
256 kfree(wq);
257
258 }
259 bitmap_free(idxd->wq_enable_map);
260
261 err_bitmap:
262 kfree(idxd->wqs);
263
264 return rc;
265 }
266
idxd_clean_engines(struct idxd_device * idxd)267 static void idxd_clean_engines(struct idxd_device *idxd)
268 {
269 struct idxd_engine *engine;
270 struct device *conf_dev;
271 int i;
272
273 for (i = 0; i < idxd->max_engines; i++) {
274 engine = idxd->engines[i];
275 conf_dev = engine_confdev(engine);
276 put_device(conf_dev);
277 kfree(engine);
278 }
279 kfree(idxd->engines);
280 }
281
idxd_setup_engines(struct idxd_device * idxd)282 static int idxd_setup_engines(struct idxd_device *idxd)
283 {
284 struct idxd_engine *engine;
285 struct device *dev = &idxd->pdev->dev;
286 struct device *conf_dev;
287 int i, rc;
288
289 idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *),
290 GFP_KERNEL, dev_to_node(dev));
291 if (!idxd->engines)
292 return -ENOMEM;
293
294 for (i = 0; i < idxd->max_engines; i++) {
295 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev));
296 if (!engine) {
297 rc = -ENOMEM;
298 goto err;
299 }
300
301 idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE);
302 conf_dev = engine_confdev(engine);
303 engine->id = i;
304 engine->idxd = idxd;
305 device_initialize(conf_dev);
306 conf_dev->parent = idxd_confdev(idxd);
307 conf_dev->bus = &dsa_bus_type;
308 conf_dev->type = &idxd_engine_device_type;
309 rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id);
310 if (rc < 0) {
311 put_device(conf_dev);
312 kfree(engine);
313 goto err;
314 }
315
316 idxd->engines[i] = engine;
317 }
318
319 return 0;
320
321 err:
322 while (--i >= 0) {
323 engine = idxd->engines[i];
324 conf_dev = engine_confdev(engine);
325 put_device(conf_dev);
326 kfree(engine);
327 }
328 kfree(idxd->engines);
329
330 return rc;
331 }
332
idxd_clean_groups(struct idxd_device * idxd)333 static void idxd_clean_groups(struct idxd_device *idxd)
334 {
335 struct idxd_group *group;
336 int i;
337
338 for (i = 0; i < idxd->max_groups; i++) {
339 group = idxd->groups[i];
340 put_device(group_confdev(group));
341 kfree(group);
342 }
343 kfree(idxd->groups);
344 }
345
idxd_setup_groups(struct idxd_device * idxd)346 static int idxd_setup_groups(struct idxd_device *idxd)
347 {
348 struct device *dev = &idxd->pdev->dev;
349 struct device *conf_dev;
350 struct idxd_group *group;
351 int i, rc;
352
353 idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *),
354 GFP_KERNEL, dev_to_node(dev));
355 if (!idxd->groups)
356 return -ENOMEM;
357
358 for (i = 0; i < idxd->max_groups; i++) {
359 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev));
360 if (!group) {
361 rc = -ENOMEM;
362 goto err;
363 }
364
365 idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP);
366 conf_dev = group_confdev(group);
367 group->id = i;
368 group->idxd = idxd;
369 device_initialize(conf_dev);
370 conf_dev->parent = idxd_confdev(idxd);
371 conf_dev->bus = &dsa_bus_type;
372 conf_dev->type = &idxd_group_device_type;
373 rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id);
374 if (rc < 0) {
375 put_device(conf_dev);
376 kfree(group);
377 goto err;
378 }
379
380 idxd->groups[i] = group;
381 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) {
382 group->tc_a = 1;
383 group->tc_b = 1;
384 } else {
385 group->tc_a = -1;
386 group->tc_b = -1;
387 }
388 /*
389 * The default value is the same as the value of
390 * total read buffers in GRPCAP.
391 */
392 group->rdbufs_allowed = idxd->max_rdbufs;
393 }
394
395 return 0;
396
397 err:
398 while (--i >= 0) {
399 group = idxd->groups[i];
400 put_device(group_confdev(group));
401 kfree(group);
402 }
403 kfree(idxd->groups);
404
405 return rc;
406 }
407
idxd_cleanup_internals(struct idxd_device * idxd)408 static void idxd_cleanup_internals(struct idxd_device *idxd)
409 {
410 idxd_clean_groups(idxd);
411 idxd_clean_engines(idxd);
412 idxd_clean_wqs(idxd);
413 destroy_workqueue(idxd->wq);
414 }
415
idxd_init_evl(struct idxd_device * idxd)416 static int idxd_init_evl(struct idxd_device *idxd)
417 {
418 struct device *dev = &idxd->pdev->dev;
419 unsigned int evl_cache_size;
420 struct idxd_evl *evl;
421 const char *idxd_name;
422
423 if (idxd->hw.gen_cap.evl_support == 0)
424 return 0;
425
426 evl = kzalloc_node(sizeof(*evl), GFP_KERNEL, dev_to_node(dev));
427 if (!evl)
428 return -ENOMEM;
429
430 mutex_init(&evl->lock);
431 evl->size = IDXD_EVL_SIZE_MIN;
432
433 idxd_name = dev_name(idxd_confdev(idxd));
434 evl_cache_size = sizeof(struct idxd_evl_fault) + evl_ent_size(idxd);
435 /*
436 * Since completion record in evl_cache will be copied to user
437 * when handling completion record page fault, need to create
438 * the cache suitable for user copy.
439 */
440 idxd->evl_cache = kmem_cache_create_usercopy(idxd_name, evl_cache_size,
441 0, 0, 0, evl_cache_size,
442 NULL);
443 if (!idxd->evl_cache) {
444 kfree(evl);
445 return -ENOMEM;
446 }
447
448 idxd->evl = evl;
449 return 0;
450 }
451
idxd_setup_internals(struct idxd_device * idxd)452 static int idxd_setup_internals(struct idxd_device *idxd)
453 {
454 struct device *dev = &idxd->pdev->dev;
455 int rc;
456
457 init_waitqueue_head(&idxd->cmd_waitq);
458
459 rc = idxd_setup_wqs(idxd);
460 if (rc < 0)
461 goto err_wqs;
462
463 rc = idxd_setup_engines(idxd);
464 if (rc < 0)
465 goto err_engine;
466
467 rc = idxd_setup_groups(idxd);
468 if (rc < 0)
469 goto err_group;
470
471 idxd->wq = create_workqueue(dev_name(dev));
472 if (!idxd->wq) {
473 rc = -ENOMEM;
474 goto err_wkq_create;
475 }
476
477 rc = idxd_init_evl(idxd);
478 if (rc < 0)
479 goto err_evl;
480
481 return 0;
482
483 err_evl:
484 destroy_workqueue(idxd->wq);
485 err_wkq_create:
486 idxd_clean_groups(idxd);
487 err_group:
488 idxd_clean_engines(idxd);
489 err_engine:
490 idxd_clean_wqs(idxd);
491 err_wqs:
492 return rc;
493 }
494
idxd_read_table_offsets(struct idxd_device * idxd)495 static void idxd_read_table_offsets(struct idxd_device *idxd)
496 {
497 union offsets_reg offsets;
498 struct device *dev = &idxd->pdev->dev;
499
500 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET);
501 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64));
502 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT;
503 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset);
504 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT;
505 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset);
506 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT;
507 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset);
508 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT;
509 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset);
510 }
511
multi_u64_to_bmap(unsigned long * bmap,u64 * val,int count)512 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count)
513 {
514 int i, j, nr;
515
516 for (i = 0, nr = 0; i < count; i++) {
517 for (j = 0; j < BITS_PER_LONG_LONG; j++) {
518 if (val[i] & BIT(j))
519 set_bit(nr, bmap);
520 nr++;
521 }
522 }
523 }
524
idxd_read_caps(struct idxd_device * idxd)525 static void idxd_read_caps(struct idxd_device *idxd)
526 {
527 struct device *dev = &idxd->pdev->dev;
528 int i;
529
530 /* reading generic capabilities */
531 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET);
532 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits);
533
534 if (idxd->hw.gen_cap.cmd_cap) {
535 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET);
536 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap);
537 }
538
539 /* reading command capabilities */
540 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE))
541 idxd->request_int_handles = true;
542
543 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift;
544 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes);
545 idxd_set_max_batch_size(idxd->data->type, idxd, 1U << idxd->hw.gen_cap.max_batch_shift);
546 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size);
547 if (idxd->hw.gen_cap.config_en)
548 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags);
549
550 /* reading group capabilities */
551 idxd->hw.group_cap.bits =
552 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET);
553 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits);
554 idxd->max_groups = idxd->hw.group_cap.num_groups;
555 dev_dbg(dev, "max groups: %u\n", idxd->max_groups);
556 idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs;
557 dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs);
558 idxd->nr_rdbufs = idxd->max_rdbufs;
559
560 /* read engine capabilities */
561 idxd->hw.engine_cap.bits =
562 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET);
563 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits);
564 idxd->max_engines = idxd->hw.engine_cap.num_engines;
565 dev_dbg(dev, "max engines: %u\n", idxd->max_engines);
566
567 /* read workqueue capabilities */
568 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET);
569 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits);
570 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size;
571 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size);
572 idxd->max_wqs = idxd->hw.wq_cap.num_wqs;
573 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs);
574 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN);
575 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size);
576
577 /* reading operation capabilities */
578 for (i = 0; i < 4; i++) {
579 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base +
580 IDXD_OPCAP_OFFSET + i * sizeof(u64));
581 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]);
582 }
583 multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4);
584
585 /* read iaa cap */
586 if (idxd->data->type == IDXD_TYPE_IAX && idxd->hw.version >= DEVICE_VERSION_2)
587 idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET);
588 }
589
idxd_free(struct idxd_device * idxd)590 static void idxd_free(struct idxd_device *idxd)
591 {
592 if (!idxd)
593 return;
594
595 put_device(idxd_confdev(idxd));
596 bitmap_free(idxd->opcap_bmap);
597 ida_free(&idxd_ida, idxd->id);
598 kfree(idxd);
599 }
600
idxd_alloc(struct pci_dev * pdev,struct idxd_driver_data * data)601 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data)
602 {
603 struct device *dev = &pdev->dev;
604 struct device *conf_dev;
605 struct idxd_device *idxd;
606 int rc;
607
608 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev));
609 if (!idxd)
610 return NULL;
611
612 conf_dev = idxd_confdev(idxd);
613 idxd->pdev = pdev;
614 idxd->data = data;
615 idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type);
616 idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL);
617 if (idxd->id < 0)
618 goto err_ida;
619
620 idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev));
621 if (!idxd->opcap_bmap)
622 goto err_opcap;
623
624 device_initialize(conf_dev);
625 conf_dev->parent = dev;
626 conf_dev->bus = &dsa_bus_type;
627 conf_dev->type = idxd->data->dev_type;
628 rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id);
629 if (rc < 0)
630 goto err_name;
631
632 spin_lock_init(&idxd->dev_lock);
633 spin_lock_init(&idxd->cmd_lock);
634
635 return idxd;
636
637 err_name:
638 put_device(conf_dev);
639 bitmap_free(idxd->opcap_bmap);
640 err_opcap:
641 ida_free(&idxd_ida, idxd->id);
642 err_ida:
643 kfree(idxd);
644
645 return NULL;
646 }
647
idxd_enable_system_pasid(struct idxd_device * idxd)648 static int idxd_enable_system_pasid(struct idxd_device *idxd)
649 {
650 struct pci_dev *pdev = idxd->pdev;
651 struct device *dev = &pdev->dev;
652 struct iommu_domain *domain;
653 ioasid_t pasid;
654 int ret;
655
656 /*
657 * Attach a global PASID to the DMA domain so that we can use ENQCMDS
658 * to submit work on buffers mapped by DMA API.
659 */
660 domain = iommu_get_domain_for_dev(dev);
661 if (!domain)
662 return -EPERM;
663
664 pasid = iommu_alloc_global_pasid(dev);
665 if (pasid == IOMMU_PASID_INVALID)
666 return -ENOSPC;
667
668 /*
669 * DMA domain is owned by the driver, it should support all valid
670 * types such as DMA-FQ, identity, etc.
671 */
672 ret = iommu_attach_device_pasid(domain, dev, pasid, NULL);
673 if (ret) {
674 dev_err(dev, "failed to attach device pasid %d, domain type %d",
675 pasid, domain->type);
676 iommu_free_global_pasid(pasid);
677 return ret;
678 }
679
680 /* Since we set user privilege for kernel DMA, enable completion IRQ */
681 idxd_set_user_intr(idxd, 1);
682 idxd->pasid = pasid;
683
684 return ret;
685 }
686
idxd_disable_system_pasid(struct idxd_device * idxd)687 static void idxd_disable_system_pasid(struct idxd_device *idxd)
688 {
689 struct pci_dev *pdev = idxd->pdev;
690 struct device *dev = &pdev->dev;
691 struct iommu_domain *domain;
692
693 domain = iommu_get_domain_for_dev(dev);
694 if (!domain)
695 return;
696
697 iommu_detach_device_pasid(domain, dev, idxd->pasid);
698 iommu_free_global_pasid(idxd->pasid);
699
700 idxd_set_user_intr(idxd, 0);
701 idxd->sva = NULL;
702 idxd->pasid = IOMMU_PASID_INVALID;
703 }
704
idxd_enable_sva(struct pci_dev * pdev)705 static int idxd_enable_sva(struct pci_dev *pdev)
706 {
707 int ret;
708
709 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF);
710 if (ret)
711 return ret;
712
713 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
714 if (ret)
715 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF);
716
717 return ret;
718 }
719
idxd_disable_sva(struct pci_dev * pdev)720 static void idxd_disable_sva(struct pci_dev *pdev)
721 {
722 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
723 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF);
724 }
725
idxd_probe(struct idxd_device * idxd)726 static int idxd_probe(struct idxd_device *idxd)
727 {
728 struct pci_dev *pdev = idxd->pdev;
729 struct device *dev = &pdev->dev;
730 int rc;
731
732 dev_dbg(dev, "%s entered and resetting device\n", __func__);
733 rc = idxd_device_init_reset(idxd);
734 if (rc < 0)
735 return rc;
736
737 dev_dbg(dev, "IDXD reset complete\n");
738
739 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) {
740 if (idxd_enable_sva(pdev)) {
741 dev_warn(dev, "Unable to turn on user SVA feature.\n");
742 } else {
743 set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
744
745 rc = idxd_enable_system_pasid(idxd);
746 if (rc)
747 dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc);
748 else
749 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
750 }
751 } else if (!sva) {
752 dev_warn(dev, "User forced SVA off via module param.\n");
753 }
754
755 idxd_read_caps(idxd);
756 idxd_read_table_offsets(idxd);
757
758 rc = idxd_setup_internals(idxd);
759 if (rc)
760 goto err;
761
762 /* If the configs are readonly, then load them from device */
763 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) {
764 dev_dbg(dev, "Loading RO device config\n");
765 rc = idxd_device_load_config(idxd);
766 if (rc < 0)
767 goto err_config;
768 }
769
770 rc = idxd_setup_interrupts(idxd);
771 if (rc)
772 goto err_config;
773
774 idxd->major = idxd_cdev_get_major(idxd);
775
776 rc = perfmon_pmu_init(idxd);
777 if (rc < 0)
778 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc);
779
780 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id);
781 return 0;
782
783 err_config:
784 idxd_cleanup_internals(idxd);
785 err:
786 if (device_pasid_enabled(idxd))
787 idxd_disable_system_pasid(idxd);
788 if (device_user_pasid_enabled(idxd))
789 idxd_disable_sva(pdev);
790 return rc;
791 }
792
idxd_cleanup(struct idxd_device * idxd)793 static void idxd_cleanup(struct idxd_device *idxd)
794 {
795 perfmon_pmu_remove(idxd);
796 idxd_cleanup_interrupts(idxd);
797 idxd_cleanup_internals(idxd);
798 if (device_pasid_enabled(idxd))
799 idxd_disable_system_pasid(idxd);
800 if (device_user_pasid_enabled(idxd))
801 idxd_disable_sva(idxd->pdev);
802 }
803
804 /*
805 * Attach IDXD device to IDXD driver.
806 */
idxd_bind(struct device_driver * drv,const char * buf)807 static int idxd_bind(struct device_driver *drv, const char *buf)
808 {
809 const struct bus_type *bus = drv->bus;
810 struct device *dev;
811 int err = -ENODEV;
812
813 dev = bus_find_device_by_name(bus, NULL, buf);
814 if (dev)
815 err = device_driver_attach(drv, dev);
816
817 put_device(dev);
818
819 return err;
820 }
821
822 /*
823 * Detach IDXD device from driver.
824 */
idxd_unbind(struct device_driver * drv,const char * buf)825 static void idxd_unbind(struct device_driver *drv, const char *buf)
826 {
827 const struct bus_type *bus = drv->bus;
828 struct device *dev;
829
830 dev = bus_find_device_by_name(bus, NULL, buf);
831 if (dev && dev->driver == drv)
832 device_release_driver(dev);
833
834 put_device(dev);
835 }
836
837 #define idxd_free_saved_configs(saved_configs, count) \
838 do { \
839 int i; \
840 \
841 for (i = 0; i < (count); i++) \
842 kfree(saved_configs[i]); \
843 } while (0)
844
idxd_free_saved(struct idxd_group ** saved_groups,struct idxd_engine ** saved_engines,struct idxd_wq ** saved_wqs,struct idxd_device * idxd)845 static void idxd_free_saved(struct idxd_group **saved_groups,
846 struct idxd_engine **saved_engines,
847 struct idxd_wq **saved_wqs,
848 struct idxd_device *idxd)
849 {
850 if (saved_groups)
851 idxd_free_saved_configs(saved_groups, idxd->max_groups);
852 if (saved_engines)
853 idxd_free_saved_configs(saved_engines, idxd->max_engines);
854 if (saved_wqs)
855 idxd_free_saved_configs(saved_wqs, idxd->max_wqs);
856 }
857
858 /*
859 * Save IDXD device configurations including engines, groups, wqs etc.
860 * The saved configurations can be restored when needed.
861 */
idxd_device_config_save(struct idxd_device * idxd,struct idxd_saved_states * idxd_saved)862 static int idxd_device_config_save(struct idxd_device *idxd,
863 struct idxd_saved_states *idxd_saved)
864 {
865 struct device *dev = &idxd->pdev->dev;
866 int i;
867
868 memcpy(&idxd_saved->saved_idxd, idxd, sizeof(*idxd));
869
870 if (idxd->evl) {
871 memcpy(&idxd_saved->saved_evl, idxd->evl,
872 sizeof(struct idxd_evl));
873 }
874
875 struct idxd_group **saved_groups __free(kfree) =
876 kcalloc_node(idxd->max_groups,
877 sizeof(struct idxd_group *),
878 GFP_KERNEL, dev_to_node(dev));
879 if (!saved_groups)
880 return -ENOMEM;
881
882 for (i = 0; i < idxd->max_groups; i++) {
883 struct idxd_group *saved_group __free(kfree) =
884 kzalloc_node(sizeof(*saved_group), GFP_KERNEL,
885 dev_to_node(dev));
886
887 if (!saved_group) {
888 /* Free saved groups */
889 idxd_free_saved(saved_groups, NULL, NULL, idxd);
890
891 return -ENOMEM;
892 }
893
894 memcpy(saved_group, idxd->groups[i], sizeof(*saved_group));
895 saved_groups[i] = no_free_ptr(saved_group);
896 }
897
898 struct idxd_engine **saved_engines =
899 kcalloc_node(idxd->max_engines,
900 sizeof(struct idxd_engine *),
901 GFP_KERNEL, dev_to_node(dev));
902 if (!saved_engines) {
903 /* Free saved groups */
904 idxd_free_saved(saved_groups, NULL, NULL, idxd);
905
906 return -ENOMEM;
907 }
908 for (i = 0; i < idxd->max_engines; i++) {
909 struct idxd_engine *saved_engine __free(kfree) =
910 kzalloc_node(sizeof(*saved_engine), GFP_KERNEL,
911 dev_to_node(dev));
912 if (!saved_engine) {
913 /* Free saved groups and engines */
914 idxd_free_saved(saved_groups, saved_engines, NULL,
915 idxd);
916
917 return -ENOMEM;
918 }
919
920 memcpy(saved_engine, idxd->engines[i], sizeof(*saved_engine));
921 saved_engines[i] = no_free_ptr(saved_engine);
922 }
923
924 unsigned long *saved_wq_enable_map __free(bitmap) =
925 bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL,
926 dev_to_node(dev));
927 if (!saved_wq_enable_map) {
928 /* Free saved groups and engines */
929 idxd_free_saved(saved_groups, saved_engines, NULL, idxd);
930
931 return -ENOMEM;
932 }
933
934 bitmap_copy(saved_wq_enable_map, idxd->wq_enable_map, idxd->max_wqs);
935
936 struct idxd_wq **saved_wqs __free(kfree) =
937 kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *),
938 GFP_KERNEL, dev_to_node(dev));
939 if (!saved_wqs) {
940 /* Free saved groups and engines */
941 idxd_free_saved(saved_groups, saved_engines, NULL, idxd);
942
943 return -ENOMEM;
944 }
945
946 for (i = 0; i < idxd->max_wqs; i++) {
947 struct idxd_wq *saved_wq __free(kfree) =
948 kzalloc_node(sizeof(*saved_wq), GFP_KERNEL,
949 dev_to_node(dev));
950 struct idxd_wq *wq;
951
952 if (!saved_wq) {
953 /* Free saved groups, engines, and wqs */
954 idxd_free_saved(saved_groups, saved_engines, saved_wqs,
955 idxd);
956
957 return -ENOMEM;
958 }
959
960 if (!test_bit(i, saved_wq_enable_map))
961 continue;
962
963 wq = idxd->wqs[i];
964 mutex_lock(&wq->wq_lock);
965 memcpy(saved_wq, wq, sizeof(*saved_wq));
966 saved_wqs[i] = no_free_ptr(saved_wq);
967 mutex_unlock(&wq->wq_lock);
968 }
969
970 /* Save configurations */
971 idxd_saved->saved_groups = no_free_ptr(saved_groups);
972 idxd_saved->saved_engines = no_free_ptr(saved_engines);
973 idxd_saved->saved_wq_enable_map = no_free_ptr(saved_wq_enable_map);
974 idxd_saved->saved_wqs = no_free_ptr(saved_wqs);
975
976 return 0;
977 }
978
979 /*
980 * Restore IDXD device configurations including engines, groups, wqs etc
981 * that were saved before.
982 */
idxd_device_config_restore(struct idxd_device * idxd,struct idxd_saved_states * idxd_saved)983 static void idxd_device_config_restore(struct idxd_device *idxd,
984 struct idxd_saved_states *idxd_saved)
985 {
986 struct idxd_evl *saved_evl = &idxd_saved->saved_evl;
987 int i;
988
989 idxd->rdbuf_limit = idxd_saved->saved_idxd.rdbuf_limit;
990
991 idxd->evl->size = saved_evl->size;
992
993 for (i = 0; i < idxd->max_groups; i++) {
994 struct idxd_group *saved_group, *group;
995
996 saved_group = idxd_saved->saved_groups[i];
997 group = idxd->groups[i];
998
999 group->rdbufs_allowed = saved_group->rdbufs_allowed;
1000 group->rdbufs_reserved = saved_group->rdbufs_reserved;
1001 group->tc_a = saved_group->tc_a;
1002 group->tc_b = saved_group->tc_b;
1003 group->use_rdbuf_limit = saved_group->use_rdbuf_limit;
1004
1005 kfree(saved_group);
1006 }
1007 kfree(idxd_saved->saved_groups);
1008
1009 for (i = 0; i < idxd->max_engines; i++) {
1010 struct idxd_engine *saved_engine, *engine;
1011
1012 saved_engine = idxd_saved->saved_engines[i];
1013 engine = idxd->engines[i];
1014
1015 engine->group = saved_engine->group;
1016
1017 kfree(saved_engine);
1018 }
1019 kfree(idxd_saved->saved_engines);
1020
1021 bitmap_copy(idxd->wq_enable_map, idxd_saved->saved_wq_enable_map,
1022 idxd->max_wqs);
1023 bitmap_free(idxd_saved->saved_wq_enable_map);
1024
1025 for (i = 0; i < idxd->max_wqs; i++) {
1026 struct idxd_wq *saved_wq, *wq;
1027 size_t len;
1028
1029 if (!test_bit(i, idxd->wq_enable_map))
1030 continue;
1031
1032 saved_wq = idxd_saved->saved_wqs[i];
1033 wq = idxd->wqs[i];
1034
1035 mutex_lock(&wq->wq_lock);
1036
1037 wq->group = saved_wq->group;
1038 wq->flags = saved_wq->flags;
1039 wq->threshold = saved_wq->threshold;
1040 wq->size = saved_wq->size;
1041 wq->priority = saved_wq->priority;
1042 wq->type = saved_wq->type;
1043 len = strlen(saved_wq->name) + 1;
1044 strscpy(wq->name, saved_wq->name, len);
1045 wq->max_xfer_bytes = saved_wq->max_xfer_bytes;
1046 wq->max_batch_size = saved_wq->max_batch_size;
1047 wq->enqcmds_retries = saved_wq->enqcmds_retries;
1048 wq->descs = saved_wq->descs;
1049 wq->idxd_chan = saved_wq->idxd_chan;
1050 len = strlen(saved_wq->driver_name) + 1;
1051 strscpy(wq->driver_name, saved_wq->driver_name, len);
1052
1053 mutex_unlock(&wq->wq_lock);
1054
1055 kfree(saved_wq);
1056 }
1057
1058 kfree(idxd_saved->saved_wqs);
1059 }
1060
idxd_reset_prepare(struct pci_dev * pdev)1061 static void idxd_reset_prepare(struct pci_dev *pdev)
1062 {
1063 struct idxd_device *idxd = pci_get_drvdata(pdev);
1064 struct device *dev = &idxd->pdev->dev;
1065 const char *idxd_name;
1066 int rc;
1067
1068 dev = &idxd->pdev->dev;
1069 idxd_name = dev_name(idxd_confdev(idxd));
1070
1071 struct idxd_saved_states *idxd_saved __free(kfree) =
1072 kzalloc_node(sizeof(*idxd_saved), GFP_KERNEL,
1073 dev_to_node(&pdev->dev));
1074 if (!idxd_saved) {
1075 dev_err(dev, "HALT: no memory\n");
1076
1077 return;
1078 }
1079
1080 /* Save IDXD configurations. */
1081 rc = idxd_device_config_save(idxd, idxd_saved);
1082 if (rc < 0) {
1083 dev_err(dev, "HALT: cannot save %s configs\n", idxd_name);
1084
1085 return;
1086 }
1087
1088 idxd->idxd_saved = no_free_ptr(idxd_saved);
1089
1090 /* Save PCI device state. */
1091 pci_save_state(idxd->pdev);
1092 }
1093
idxd_reset_done(struct pci_dev * pdev)1094 static void idxd_reset_done(struct pci_dev *pdev)
1095 {
1096 struct idxd_device *idxd = pci_get_drvdata(pdev);
1097 const char *idxd_name;
1098 struct device *dev;
1099 int rc, i;
1100
1101 if (!idxd->idxd_saved)
1102 return;
1103
1104 dev = &idxd->pdev->dev;
1105 idxd_name = dev_name(idxd_confdev(idxd));
1106
1107 /* Restore PCI device state. */
1108 pci_restore_state(idxd->pdev);
1109
1110 /* Unbind idxd device from driver. */
1111 idxd_unbind(&idxd_drv.drv, idxd_name);
1112
1113 /*
1114 * Probe PCI device without allocating or changing
1115 * idxd software data which keeps the same as before FLR.
1116 */
1117 idxd_pci_probe_alloc(idxd, NULL, NULL);
1118
1119 /* Restore IDXD configurations. */
1120 idxd_device_config_restore(idxd, idxd->idxd_saved);
1121
1122 /* Re-configure IDXD device if allowed. */
1123 if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) {
1124 rc = idxd_device_config(idxd);
1125 if (rc < 0) {
1126 dev_err(dev, "HALT: %s config fails\n", idxd_name);
1127 goto out;
1128 }
1129 }
1130
1131 /* Bind IDXD device to driver. */
1132 rc = idxd_bind(&idxd_drv.drv, idxd_name);
1133 if (rc < 0) {
1134 dev_err(dev, "HALT: binding %s to driver fails\n", idxd_name);
1135 goto out;
1136 }
1137
1138 /* Bind enabled wq in the IDXD device to driver. */
1139 for (i = 0; i < idxd->max_wqs; i++) {
1140 if (test_bit(i, idxd->wq_enable_map)) {
1141 struct idxd_wq *wq = idxd->wqs[i];
1142 char wq_name[32];
1143
1144 wq->state = IDXD_WQ_DISABLED;
1145 sprintf(wq_name, "wq%d.%d", idxd->id, wq->id);
1146 /*
1147 * Bind to user driver depending on wq type.
1148 *
1149 * Currently only support user type WQ. Will support
1150 * kernel type WQ in the future.
1151 */
1152 if (wq->type == IDXD_WQT_USER)
1153 rc = idxd_bind(&idxd_user_drv.drv, wq_name);
1154 else
1155 rc = -EINVAL;
1156 if (rc < 0) {
1157 clear_bit(i, idxd->wq_enable_map);
1158 dev_err(dev,
1159 "HALT: unable to re-enable wq %s\n",
1160 dev_name(wq_confdev(wq)));
1161 }
1162 }
1163 }
1164 out:
1165 kfree(idxd->idxd_saved);
1166 }
1167
1168 static const struct pci_error_handlers idxd_error_handler = {
1169 .reset_prepare = idxd_reset_prepare,
1170 .reset_done = idxd_reset_done,
1171 };
1172
1173 /*
1174 * Probe idxd PCI device.
1175 * If idxd is not given, need to allocate idxd and set up its data.
1176 *
1177 * If idxd is given, idxd was allocated and setup already. Just need to
1178 * configure device without re-allocating and re-configuring idxd data.
1179 * This is useful for recovering from FLR.
1180 */
idxd_pci_probe_alloc(struct idxd_device * idxd,struct pci_dev * pdev,const struct pci_device_id * id)1181 int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev,
1182 const struct pci_device_id *id)
1183 {
1184 bool alloc_idxd = idxd ? false : true;
1185 struct idxd_driver_data *data;
1186 struct device *dev;
1187 int rc;
1188
1189 pdev = idxd ? idxd->pdev : pdev;
1190 dev = &pdev->dev;
1191 data = id ? (struct idxd_driver_data *)id->driver_data : NULL;
1192 rc = pci_enable_device(pdev);
1193 if (rc)
1194 return rc;
1195
1196 if (alloc_idxd) {
1197 dev_dbg(dev, "Alloc IDXD context\n");
1198 idxd = idxd_alloc(pdev, data);
1199 if (!idxd) {
1200 rc = -ENOMEM;
1201 goto err_idxd_alloc;
1202 }
1203
1204 dev_dbg(dev, "Mapping BARs\n");
1205 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0);
1206 if (!idxd->reg_base) {
1207 rc = -ENOMEM;
1208 goto err_iomap;
1209 }
1210
1211 dev_dbg(dev, "Set DMA masks\n");
1212 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1213 if (rc)
1214 goto err;
1215 }
1216
1217 dev_dbg(dev, "Set PCI master\n");
1218 pci_set_master(pdev);
1219 pci_set_drvdata(pdev, idxd);
1220
1221 if (alloc_idxd) {
1222 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET);
1223 rc = idxd_probe(idxd);
1224 if (rc) {
1225 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n");
1226 goto err;
1227 }
1228
1229 if (data->load_device_defaults) {
1230 rc = data->load_device_defaults(idxd);
1231 if (rc)
1232 dev_warn(dev, "IDXD loading device defaults failed\n");
1233 }
1234
1235 rc = idxd_register_devices(idxd);
1236 if (rc) {
1237 dev_err(dev, "IDXD sysfs setup failed\n");
1238 goto err_dev_register;
1239 }
1240
1241 rc = idxd_device_init_debugfs(idxd);
1242 if (rc)
1243 dev_warn(dev, "IDXD debugfs failed to setup\n");
1244 }
1245
1246 if (!alloc_idxd) {
1247 /* Release interrupts in the IDXD device. */
1248 idxd_cleanup_interrupts(idxd);
1249
1250 /* Re-enable interrupts in the IDXD device. */
1251 rc = idxd_setup_interrupts(idxd);
1252 if (rc)
1253 dev_warn(dev, "IDXD interrupts failed to setup\n");
1254 }
1255
1256 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n",
1257 idxd->hw.version);
1258
1259 if (data)
1260 idxd->user_submission_safe = data->user_submission_safe;
1261
1262 return 0;
1263
1264 err_dev_register:
1265 idxd_cleanup(idxd);
1266 err:
1267 pci_iounmap(pdev, idxd->reg_base);
1268 err_iomap:
1269 idxd_free(idxd);
1270 err_idxd_alloc:
1271 pci_disable_device(pdev);
1272 return rc;
1273 }
1274
idxd_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)1275 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1276 {
1277 return idxd_pci_probe_alloc(NULL, pdev, id);
1278 }
1279
idxd_wqs_quiesce(struct idxd_device * idxd)1280 void idxd_wqs_quiesce(struct idxd_device *idxd)
1281 {
1282 struct idxd_wq *wq;
1283 int i;
1284
1285 for (i = 0; i < idxd->max_wqs; i++) {
1286 wq = idxd->wqs[i];
1287 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL)
1288 idxd_wq_quiesce(wq);
1289 }
1290 }
1291
idxd_shutdown(struct pci_dev * pdev)1292 static void idxd_shutdown(struct pci_dev *pdev)
1293 {
1294 struct idxd_device *idxd = pci_get_drvdata(pdev);
1295 struct idxd_irq_entry *irq_entry;
1296 int rc;
1297
1298 rc = idxd_device_disable(idxd);
1299 if (rc)
1300 dev_err(&pdev->dev, "Disabling device failed\n");
1301
1302 irq_entry = &idxd->ie;
1303 synchronize_irq(irq_entry->vector);
1304 idxd_mask_error_interrupts(idxd);
1305 flush_workqueue(idxd->wq);
1306 }
1307
idxd_remove(struct pci_dev * pdev)1308 static void idxd_remove(struct pci_dev *pdev)
1309 {
1310 struct idxd_device *idxd = pci_get_drvdata(pdev);
1311
1312 idxd_unregister_devices(idxd);
1313 /*
1314 * When ->release() is called for the idxd->conf_dev, it frees all the memory related
1315 * to the idxd context. The driver still needs those bits in order to do the rest of
1316 * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref
1317 * on the device here to hold off the freeing while allowing the idxd sub-driver
1318 * to unbind.
1319 */
1320 get_device(idxd_confdev(idxd));
1321 device_unregister(idxd_confdev(idxd));
1322 idxd_shutdown(pdev);
1323 idxd_device_remove_debugfs(idxd);
1324 idxd_cleanup(idxd);
1325 pci_iounmap(pdev, idxd->reg_base);
1326 put_device(idxd_confdev(idxd));
1327 idxd_free(idxd);
1328 pci_disable_device(pdev);
1329 }
1330
1331 static struct pci_driver idxd_pci_driver = {
1332 .name = DRV_NAME,
1333 .id_table = idxd_pci_tbl,
1334 .probe = idxd_pci_probe,
1335 .remove = idxd_remove,
1336 .shutdown = idxd_shutdown,
1337 .err_handler = &idxd_error_handler,
1338 };
1339
idxd_init_module(void)1340 static int __init idxd_init_module(void)
1341 {
1342 int err;
1343
1344 /*
1345 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in
1346 * enumerating the device. We can not utilize it.
1347 */
1348 if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) {
1349 pr_warn("idxd driver failed to load without MOVDIR64B.\n");
1350 return -ENODEV;
1351 }
1352
1353 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD))
1354 pr_warn("Platform does not have ENQCMD(S) support.\n");
1355 else
1356 support_enqcmd = true;
1357
1358 err = idxd_driver_register(&idxd_drv);
1359 if (err < 0)
1360 goto err_idxd_driver_register;
1361
1362 err = idxd_driver_register(&idxd_dmaengine_drv);
1363 if (err < 0)
1364 goto err_idxd_dmaengine_driver_register;
1365
1366 err = idxd_driver_register(&idxd_user_drv);
1367 if (err < 0)
1368 goto err_idxd_user_driver_register;
1369
1370 err = idxd_cdev_register();
1371 if (err)
1372 goto err_cdev_register;
1373
1374 err = idxd_init_debugfs();
1375 if (err)
1376 goto err_debugfs;
1377
1378 err = pci_register_driver(&idxd_pci_driver);
1379 if (err)
1380 goto err_pci_register;
1381
1382 return 0;
1383
1384 err_pci_register:
1385 idxd_remove_debugfs();
1386 err_debugfs:
1387 idxd_cdev_remove();
1388 err_cdev_register:
1389 idxd_driver_unregister(&idxd_user_drv);
1390 err_idxd_user_driver_register:
1391 idxd_driver_unregister(&idxd_dmaengine_drv);
1392 err_idxd_dmaengine_driver_register:
1393 idxd_driver_unregister(&idxd_drv);
1394 err_idxd_driver_register:
1395 return err;
1396 }
1397 module_init(idxd_init_module);
1398
idxd_exit_module(void)1399 static void __exit idxd_exit_module(void)
1400 {
1401 idxd_driver_unregister(&idxd_user_drv);
1402 idxd_driver_unregister(&idxd_dmaengine_drv);
1403 idxd_driver_unregister(&idxd_drv);
1404 pci_unregister_driver(&idxd_pci_driver);
1405 idxd_cdev_remove();
1406 idxd_remove_debugfs();
1407 }
1408 module_exit(idxd_exit_module);
1409