1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3
4 #include <crypto/algapi.h>
5 #include <linux/module.h>
6 #include <linux/mutex.h>
7 #include <linux/slab.h>
8 #include <linux/fs.h>
9 #include <linux/bitops.h>
10 #include <linux/pci.h>
11 #include <linux/cdev.h>
12 #include <linux/uaccess.h>
13
14 #include "adf_accel_devices.h"
15 #include "adf_common_drv.h"
16 #include "adf_cfg.h"
17 #include "adf_cfg_common.h"
18 #include "adf_cfg_user.h"
19
20 #define ADF_CFG_MAX_SECTION 512
21 #define ADF_CFG_MAX_KEY_VAL 256
22
23 #define DEVICE_NAME "qat_adf_ctl"
24
25 static DEFINE_MUTEX(adf_ctl_lock);
26 static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg);
27
28 static const struct file_operations adf_ctl_ops = {
29 .owner = THIS_MODULE,
30 .unlocked_ioctl = adf_ctl_ioctl,
31 .compat_ioctl = compat_ptr_ioctl,
32 };
33
34 static const struct class adf_ctl_class = {
35 .name = DEVICE_NAME,
36 };
37
38 struct adf_ctl_drv_info {
39 unsigned int major;
40 struct cdev drv_cdev;
41 };
42
43 static struct adf_ctl_drv_info adf_ctl_drv;
44
adf_chr_drv_destroy(void)45 static void adf_chr_drv_destroy(void)
46 {
47 device_destroy(&adf_ctl_class, MKDEV(adf_ctl_drv.major, 0));
48 cdev_del(&adf_ctl_drv.drv_cdev);
49 class_unregister(&adf_ctl_class);
50 unregister_chrdev_region(MKDEV(adf_ctl_drv.major, 0), 1);
51 }
52
adf_chr_drv_create(void)53 static int adf_chr_drv_create(void)
54 {
55 dev_t dev_id;
56 struct device *drv_device;
57 int ret;
58
59 if (alloc_chrdev_region(&dev_id, 0, 1, DEVICE_NAME)) {
60 pr_err("QAT: unable to allocate chrdev region\n");
61 return -EFAULT;
62 }
63
64 ret = class_register(&adf_ctl_class);
65 if (ret)
66 goto err_chrdev_unreg;
67
68 adf_ctl_drv.major = MAJOR(dev_id);
69 cdev_init(&adf_ctl_drv.drv_cdev, &adf_ctl_ops);
70 if (cdev_add(&adf_ctl_drv.drv_cdev, dev_id, 1)) {
71 pr_err("QAT: cdev add failed\n");
72 goto err_class_destr;
73 }
74
75 drv_device = device_create(&adf_ctl_class, NULL,
76 MKDEV(adf_ctl_drv.major, 0),
77 NULL, DEVICE_NAME);
78 if (IS_ERR(drv_device)) {
79 pr_err("QAT: failed to create device\n");
80 goto err_cdev_del;
81 }
82 return 0;
83 err_cdev_del:
84 cdev_del(&adf_ctl_drv.drv_cdev);
85 err_class_destr:
86 class_unregister(&adf_ctl_class);
87 err_chrdev_unreg:
88 unregister_chrdev_region(dev_id, 1);
89 return -EFAULT;
90 }
91
adf_ctl_alloc_resources(unsigned long arg)92 static struct adf_user_cfg_ctl_data *adf_ctl_alloc_resources(unsigned long arg)
93 {
94 struct adf_user_cfg_ctl_data *cfg_data;
95
96 cfg_data = memdup_user((void __user *)arg, sizeof(*cfg_data));
97 if (IS_ERR(cfg_data))
98 pr_err("QAT: failed to copy from user cfg_data.\n");
99 return cfg_data;
100 }
101
adf_add_key_value_data(struct adf_accel_dev * accel_dev,const char * section,const struct adf_user_cfg_key_val * key_val)102 static int adf_add_key_value_data(struct adf_accel_dev *accel_dev,
103 const char *section,
104 const struct adf_user_cfg_key_val *key_val)
105 {
106 if (key_val->type == ADF_HEX) {
107 long *ptr = (long *)key_val->val;
108 long val = *ptr;
109
110 if (adf_cfg_add_key_value_param(accel_dev, section,
111 key_val->key, (void *)val,
112 key_val->type)) {
113 dev_err(&GET_DEV(accel_dev),
114 "failed to add hex keyvalue.\n");
115 return -EFAULT;
116 }
117 } else {
118 if (adf_cfg_add_key_value_param(accel_dev, section,
119 key_val->key, key_val->val,
120 key_val->type)) {
121 dev_err(&GET_DEV(accel_dev),
122 "failed to add keyvalue.\n");
123 return -EFAULT;
124 }
125 }
126 return 0;
127 }
128
adf_copy_key_value_data(struct adf_accel_dev * accel_dev,struct adf_user_cfg_ctl_data * ctl_data)129 static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev,
130 struct adf_user_cfg_ctl_data *ctl_data)
131 {
132 struct adf_user_cfg_key_val key_val;
133 struct adf_user_cfg_key_val *params_head;
134 struct adf_user_cfg_section section, *section_head;
135 int i, j;
136
137 section_head = ctl_data->config_section;
138
139 for (i = 0; section_head && i < ADF_CFG_MAX_SECTION; i++) {
140 if (copy_from_user(§ion, (void __user *)section_head,
141 sizeof(*section_head))) {
142 dev_err(&GET_DEV(accel_dev),
143 "failed to copy section info\n");
144 goto out_err;
145 }
146
147 if (adf_cfg_section_add(accel_dev, section.name)) {
148 dev_err(&GET_DEV(accel_dev),
149 "failed to add section.\n");
150 goto out_err;
151 }
152
153 params_head = section.params;
154
155 for (j = 0; params_head && j < ADF_CFG_MAX_KEY_VAL; j++) {
156 if (copy_from_user(&key_val, (void __user *)params_head,
157 sizeof(key_val))) {
158 dev_err(&GET_DEV(accel_dev),
159 "Failed to copy keyvalue.\n");
160 goto out_err;
161 }
162 if (adf_add_key_value_data(accel_dev, section.name,
163 &key_val)) {
164 goto out_err;
165 }
166 params_head = key_val.next;
167 }
168 section_head = section.next;
169 }
170 return 0;
171 out_err:
172 adf_cfg_del_all(accel_dev);
173 return -EFAULT;
174 }
175
adf_ctl_ioctl_dev_config(struct file * fp,unsigned int cmd,unsigned long arg)176 static int adf_ctl_ioctl_dev_config(struct file *fp, unsigned int cmd,
177 unsigned long arg)
178 {
179 struct adf_user_cfg_ctl_data *ctl_data;
180 struct adf_accel_dev *accel_dev;
181 int ret = 0;
182
183 ctl_data = adf_ctl_alloc_resources(arg);
184 if (IS_ERR(ctl_data))
185 return PTR_ERR(ctl_data);
186
187 accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id);
188 if (!accel_dev) {
189 ret = -EFAULT;
190 goto out;
191 }
192
193 if (adf_dev_started(accel_dev)) {
194 ret = -EFAULT;
195 goto out;
196 }
197
198 if (adf_copy_key_value_data(accel_dev, ctl_data)) {
199 ret = -EFAULT;
200 goto out;
201 }
202 set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
203 out:
204 kfree(ctl_data);
205 return ret;
206 }
207
adf_ctl_is_device_in_use(int id)208 static int adf_ctl_is_device_in_use(int id)
209 {
210 struct adf_accel_dev *dev;
211
212 list_for_each_entry(dev, adf_devmgr_get_head(), list) {
213 if (id == dev->accel_id || id == ADF_CFG_ALL_DEVICES) {
214 if (adf_devmgr_in_reset(dev) || adf_dev_in_use(dev)) {
215 dev_info(&GET_DEV(dev),
216 "device qat_dev%d is busy\n",
217 dev->accel_id);
218 return -EBUSY;
219 }
220 }
221 }
222 return 0;
223 }
224
adf_ctl_stop_devices(u32 id)225 static void adf_ctl_stop_devices(u32 id)
226 {
227 struct adf_accel_dev *accel_dev;
228
229 list_for_each_entry(accel_dev, adf_devmgr_get_head(), list) {
230 if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) {
231 if (!adf_dev_started(accel_dev))
232 continue;
233
234 /* First stop all VFs */
235 if (!accel_dev->is_vf)
236 continue;
237
238 adf_dev_down(accel_dev);
239 }
240 }
241
242 list_for_each_entry(accel_dev, adf_devmgr_get_head(), list) {
243 if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) {
244 if (!adf_dev_started(accel_dev))
245 continue;
246
247 adf_dev_down(accel_dev);
248 }
249 }
250 }
251
adf_ctl_ioctl_dev_stop(struct file * fp,unsigned int cmd,unsigned long arg)252 static int adf_ctl_ioctl_dev_stop(struct file *fp, unsigned int cmd,
253 unsigned long arg)
254 {
255 int ret;
256 struct adf_user_cfg_ctl_data *ctl_data;
257
258 ctl_data = adf_ctl_alloc_resources(arg);
259 if (IS_ERR(ctl_data))
260 return PTR_ERR(ctl_data);
261
262 if (adf_devmgr_verify_id(ctl_data->device_id)) {
263 pr_err("QAT: Device %d not found\n", ctl_data->device_id);
264 ret = -ENODEV;
265 goto out;
266 }
267
268 ret = adf_ctl_is_device_in_use(ctl_data->device_id);
269 if (ret)
270 goto out;
271
272 if (ctl_data->device_id == ADF_CFG_ALL_DEVICES)
273 pr_info("QAT: Stopping all acceleration devices.\n");
274 else
275 pr_info("QAT: Stopping acceleration device qat_dev%d.\n",
276 ctl_data->device_id);
277
278 adf_ctl_stop_devices(ctl_data->device_id);
279
280 out:
281 kfree(ctl_data);
282 return ret;
283 }
284
adf_ctl_ioctl_dev_start(struct file * fp,unsigned int cmd,unsigned long arg)285 static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd,
286 unsigned long arg)
287 {
288 int ret;
289 struct adf_user_cfg_ctl_data *ctl_data;
290 struct adf_accel_dev *accel_dev;
291
292 ctl_data = adf_ctl_alloc_resources(arg);
293 if (IS_ERR(ctl_data))
294 return PTR_ERR(ctl_data);
295
296 ret = -ENODEV;
297 accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id);
298 if (!accel_dev)
299 goto out;
300
301 dev_info(&GET_DEV(accel_dev),
302 "Starting acceleration device qat_dev%d.\n",
303 ctl_data->device_id);
304
305 ret = adf_dev_up(accel_dev, false);
306
307 if (ret) {
308 dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n",
309 ctl_data->device_id);
310 adf_dev_down(accel_dev);
311 }
312 out:
313 kfree(ctl_data);
314 return ret;
315 }
316
adf_ctl_ioctl_get_num_devices(struct file * fp,unsigned int cmd,unsigned long arg)317 static int adf_ctl_ioctl_get_num_devices(struct file *fp, unsigned int cmd,
318 unsigned long arg)
319 {
320 u32 num_devices = 0;
321
322 adf_devmgr_get_num_dev(&num_devices);
323 if (copy_to_user((void __user *)arg, &num_devices, sizeof(num_devices)))
324 return -EFAULT;
325
326 return 0;
327 }
328
adf_ctl_ioctl_get_status(struct file * fp,unsigned int cmd,unsigned long arg)329 static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd,
330 unsigned long arg)
331 {
332 struct adf_hw_device_data *hw_data;
333 struct adf_dev_status_info dev_info;
334 struct adf_accel_dev *accel_dev;
335
336 if (copy_from_user(&dev_info, (void __user *)arg,
337 sizeof(struct adf_dev_status_info))) {
338 pr_err("QAT: failed to copy from user.\n");
339 return -EFAULT;
340 }
341
342 accel_dev = adf_devmgr_get_dev_by_id(dev_info.accel_id);
343 if (!accel_dev)
344 return -ENODEV;
345
346 hw_data = accel_dev->hw_device;
347 dev_info.state = adf_dev_started(accel_dev) ? DEV_UP : DEV_DOWN;
348 dev_info.num_ae = hw_data->get_num_aes(hw_data);
349 dev_info.num_accel = hw_data->get_num_accels(hw_data);
350 dev_info.num_logical_accel = hw_data->num_logical_accel;
351 dev_info.banks_per_accel = hw_data->num_banks
352 / hw_data->num_logical_accel;
353 strscpy(dev_info.name, hw_data->dev_class->name, sizeof(dev_info.name));
354 dev_info.instance_id = hw_data->instance_id;
355 dev_info.type = hw_data->dev_class->type;
356 dev_info.bus = accel_to_pci_dev(accel_dev)->bus->number;
357 dev_info.dev = PCI_SLOT(accel_to_pci_dev(accel_dev)->devfn);
358 dev_info.fun = PCI_FUNC(accel_to_pci_dev(accel_dev)->devfn);
359
360 if (copy_to_user((void __user *)arg, &dev_info,
361 sizeof(struct adf_dev_status_info))) {
362 dev_err(&GET_DEV(accel_dev), "failed to copy status.\n");
363 return -EFAULT;
364 }
365 return 0;
366 }
367
adf_ctl_ioctl(struct file * fp,unsigned int cmd,unsigned long arg)368 static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
369 {
370 int ret;
371
372 if (mutex_lock_interruptible(&adf_ctl_lock))
373 return -EFAULT;
374
375 switch (cmd) {
376 case IOCTL_CONFIG_SYS_RESOURCE_PARAMETERS:
377 ret = adf_ctl_ioctl_dev_config(fp, cmd, arg);
378 break;
379
380 case IOCTL_STOP_ACCEL_DEV:
381 ret = adf_ctl_ioctl_dev_stop(fp, cmd, arg);
382 break;
383
384 case IOCTL_START_ACCEL_DEV:
385 ret = adf_ctl_ioctl_dev_start(fp, cmd, arg);
386 break;
387
388 case IOCTL_GET_NUM_DEVICES:
389 ret = adf_ctl_ioctl_get_num_devices(fp, cmd, arg);
390 break;
391
392 case IOCTL_STATUS_ACCEL_DEV:
393 ret = adf_ctl_ioctl_get_status(fp, cmd, arg);
394 break;
395 default:
396 pr_err_ratelimited("QAT: Invalid ioctl %d\n", cmd);
397 ret = -EFAULT;
398 break;
399 }
400 mutex_unlock(&adf_ctl_lock);
401 return ret;
402 }
403
adf_register_ctl_device_driver(void)404 static int __init adf_register_ctl_device_driver(void)
405 {
406 if (adf_chr_drv_create())
407 goto err_chr_dev;
408
409 if (adf_init_misc_wq())
410 goto err_misc_wq;
411
412 if (adf_init_aer())
413 goto err_aer;
414
415 if (adf_init_pf_wq())
416 goto err_pf_wq;
417
418 if (adf_init_vf_wq())
419 goto err_vf_wq;
420
421 if (qat_crypto_register())
422 goto err_crypto_register;
423
424 if (qat_compression_register())
425 goto err_compression_register;
426
427 return 0;
428
429 err_compression_register:
430 qat_crypto_unregister();
431 err_crypto_register:
432 adf_exit_vf_wq();
433 err_vf_wq:
434 adf_exit_pf_wq();
435 err_pf_wq:
436 adf_exit_aer();
437 err_aer:
438 adf_exit_misc_wq();
439 err_misc_wq:
440 adf_chr_drv_destroy();
441 err_chr_dev:
442 mutex_destroy(&adf_ctl_lock);
443 return -EFAULT;
444 }
445
adf_unregister_ctl_device_driver(void)446 static void __exit adf_unregister_ctl_device_driver(void)
447 {
448 adf_chr_drv_destroy();
449 adf_exit_misc_wq();
450 adf_exit_aer();
451 adf_exit_vf_wq();
452 adf_exit_pf_wq();
453 qat_crypto_unregister();
454 qat_compression_unregister();
455 adf_clean_vf_map(false);
456 mutex_destroy(&adf_ctl_lock);
457 }
458
459 module_init(adf_register_ctl_device_driver);
460 module_exit(adf_unregister_ctl_device_driver);
461 MODULE_LICENSE("Dual BSD/GPL");
462 MODULE_AUTHOR("Intel");
463 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
464 MODULE_ALIAS_CRYPTO("intel_qat");
465 MODULE_VERSION(ADF_DRV_VERSION);
466 MODULE_IMPORT_NS("CRYPTO_INTERNAL");
467