1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PCI Endpoint *Function* (EPF) library
4  *
5  * Copyright (C) 2017 Texas Instruments
6  * Author: Kishon Vijay Abraham I <kishon@ti.com>
7  */
8 
9 #include <linux/device.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/slab.h>
12 #include <linux/module.h>
13 
14 #include <linux/pci-epc.h>
15 #include <linux/pci-epf.h>
16 #include <linux/pci-ep-cfs.h>
17 
18 static DEFINE_MUTEX(pci_epf_mutex);
19 
20 static const struct bus_type pci_epf_bus_type;
21 static const struct device_type pci_epf_type;
22 
23 /**
24  * pci_epf_unbind() - Notify the function driver that the binding between the
25  *		      EPF device and EPC device has been lost
26  * @epf: the EPF device which has lost the binding with the EPC device
27  *
28  * Invoke to notify the function driver that the binding between the EPF device
29  * and EPC device has been lost.
30  */
pci_epf_unbind(struct pci_epf * epf)31 void pci_epf_unbind(struct pci_epf *epf)
32 {
33 	struct pci_epf *epf_vf;
34 
35 	if (!epf->driver) {
36 		dev_WARN(&epf->dev, "epf device not bound to driver\n");
37 		return;
38 	}
39 
40 	mutex_lock(&epf->lock);
41 	list_for_each_entry(epf_vf, &epf->pci_vepf, list) {
42 		if (epf_vf->is_bound)
43 			epf_vf->driver->ops->unbind(epf_vf);
44 	}
45 	if (epf->is_bound)
46 		epf->driver->ops->unbind(epf);
47 	mutex_unlock(&epf->lock);
48 	module_put(epf->driver->owner);
49 }
50 EXPORT_SYMBOL_GPL(pci_epf_unbind);
51 
52 /**
53  * pci_epf_bind() - Notify the function driver that the EPF device has been
54  *		    bound to a EPC device
55  * @epf: the EPF device which has been bound to the EPC device
56  *
57  * Invoke to notify the function driver that it has been bound to a EPC device
58  */
pci_epf_bind(struct pci_epf * epf)59 int pci_epf_bind(struct pci_epf *epf)
60 {
61 	struct device *dev = &epf->dev;
62 	struct pci_epf *epf_vf;
63 	u8 func_no, vfunc_no;
64 	struct pci_epc *epc;
65 	int ret;
66 
67 	if (!epf->driver) {
68 		dev_WARN(dev, "epf device not bound to driver\n");
69 		return -EINVAL;
70 	}
71 
72 	if (!try_module_get(epf->driver->owner))
73 		return -EAGAIN;
74 
75 	mutex_lock(&epf->lock);
76 	list_for_each_entry(epf_vf, &epf->pci_vepf, list) {
77 		vfunc_no = epf_vf->vfunc_no;
78 
79 		if (vfunc_no < 1) {
80 			dev_err(dev, "Invalid virtual function number\n");
81 			ret = -EINVAL;
82 			goto ret;
83 		}
84 
85 		epc = epf->epc;
86 		func_no = epf->func_no;
87 		if (!IS_ERR_OR_NULL(epc)) {
88 			if (!epc->max_vfs) {
89 				dev_err(dev, "No support for virt function\n");
90 				ret = -EINVAL;
91 				goto ret;
92 			}
93 
94 			if (vfunc_no > epc->max_vfs[func_no]) {
95 				dev_err(dev, "PF%d: Exceeds max vfunc number\n",
96 					func_no);
97 				ret = -EINVAL;
98 				goto ret;
99 			}
100 		}
101 
102 		epc = epf->sec_epc;
103 		func_no = epf->sec_epc_func_no;
104 		if (!IS_ERR_OR_NULL(epc)) {
105 			if (!epc->max_vfs) {
106 				dev_err(dev, "No support for virt function\n");
107 				ret = -EINVAL;
108 				goto ret;
109 			}
110 
111 			if (vfunc_no > epc->max_vfs[func_no]) {
112 				dev_err(dev, "PF%d: Exceeds max vfunc number\n",
113 					func_no);
114 				ret = -EINVAL;
115 				goto ret;
116 			}
117 		}
118 
119 		epf_vf->func_no = epf->func_no;
120 		epf_vf->sec_epc_func_no = epf->sec_epc_func_no;
121 		epf_vf->epc = epf->epc;
122 		epf_vf->sec_epc = epf->sec_epc;
123 		ret = epf_vf->driver->ops->bind(epf_vf);
124 		if (ret)
125 			goto ret;
126 		epf_vf->is_bound = true;
127 	}
128 
129 	ret = epf->driver->ops->bind(epf);
130 	if (ret)
131 		goto ret;
132 	epf->is_bound = true;
133 
134 	mutex_unlock(&epf->lock);
135 	return 0;
136 
137 ret:
138 	mutex_unlock(&epf->lock);
139 	pci_epf_unbind(epf);
140 
141 	return ret;
142 }
143 EXPORT_SYMBOL_GPL(pci_epf_bind);
144 
145 /**
146  * pci_epf_add_vepf() - associate virtual EP function to physical EP function
147  * @epf_pf: the physical EP function to which the virtual EP function should be
148  *   associated
149  * @epf_vf: the virtual EP function to be added
150  *
151  * A physical endpoint function can be associated with multiple virtual
152  * endpoint functions. Invoke pci_epf_add_epf() to add a virtual PCI endpoint
153  * function to a physical PCI endpoint function.
154  */
pci_epf_add_vepf(struct pci_epf * epf_pf,struct pci_epf * epf_vf)155 int pci_epf_add_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf)
156 {
157 	u32 vfunc_no;
158 
159 	if (IS_ERR_OR_NULL(epf_pf) || IS_ERR_OR_NULL(epf_vf))
160 		return -EINVAL;
161 
162 	if (epf_pf->epc || epf_vf->epc || epf_vf->epf_pf)
163 		return -EBUSY;
164 
165 	if (epf_pf->sec_epc || epf_vf->sec_epc)
166 		return -EBUSY;
167 
168 	mutex_lock(&epf_pf->lock);
169 	vfunc_no = find_first_zero_bit(&epf_pf->vfunction_num_map,
170 				       BITS_PER_LONG);
171 	if (vfunc_no >= BITS_PER_LONG) {
172 		mutex_unlock(&epf_pf->lock);
173 		return -EINVAL;
174 	}
175 
176 	set_bit(vfunc_no, &epf_pf->vfunction_num_map);
177 	epf_vf->vfunc_no = vfunc_no;
178 
179 	epf_vf->epf_pf = epf_pf;
180 	epf_vf->is_vf = true;
181 
182 	list_add_tail(&epf_vf->list, &epf_pf->pci_vepf);
183 	mutex_unlock(&epf_pf->lock);
184 
185 	return 0;
186 }
187 EXPORT_SYMBOL_GPL(pci_epf_add_vepf);
188 
189 /**
190  * pci_epf_remove_vepf() - remove virtual EP function from physical EP function
191  * @epf_pf: the physical EP function from which the virtual EP function should
192  *   be removed
193  * @epf_vf: the virtual EP function to be removed
194  *
195  * Invoke to remove a virtual endpoint function from the physical endpoint
196  * function.
197  */
pci_epf_remove_vepf(struct pci_epf * epf_pf,struct pci_epf * epf_vf)198 void pci_epf_remove_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf)
199 {
200 	if (IS_ERR_OR_NULL(epf_pf) || IS_ERR_OR_NULL(epf_vf))
201 		return;
202 
203 	mutex_lock(&epf_pf->lock);
204 	clear_bit(epf_vf->vfunc_no, &epf_pf->vfunction_num_map);
205 	epf_vf->epf_pf = NULL;
206 	list_del(&epf_vf->list);
207 	mutex_unlock(&epf_pf->lock);
208 }
209 EXPORT_SYMBOL_GPL(pci_epf_remove_vepf);
210 
211 /**
212  * pci_epf_free_space() - free the allocated PCI EPF register space
213  * @epf: the EPF device from whom to free the memory
214  * @addr: the virtual address of the PCI EPF register space
215  * @bar: the BAR number corresponding to the register space
216  * @type: Identifies if the allocated space is for primary EPC or secondary EPC
217  *
218  * Invoke to free the allocated PCI EPF register space.
219  */
pci_epf_free_space(struct pci_epf * epf,void * addr,enum pci_barno bar,enum pci_epc_interface_type type)220 void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar,
221 			enum pci_epc_interface_type type)
222 {
223 	struct device *dev;
224 	struct pci_epf_bar *epf_bar;
225 	struct pci_epc *epc;
226 
227 	if (!addr)
228 		return;
229 
230 	if (type == PRIMARY_INTERFACE) {
231 		epc = epf->epc;
232 		epf_bar = epf->bar;
233 	} else {
234 		epc = epf->sec_epc;
235 		epf_bar = epf->sec_epc_bar;
236 	}
237 
238 	dev = epc->dev.parent;
239 	dma_free_coherent(dev, epf_bar[bar].size, addr,
240 			  epf_bar[bar].phys_addr);
241 
242 	epf_bar[bar].phys_addr = 0;
243 	epf_bar[bar].addr = NULL;
244 	epf_bar[bar].size = 0;
245 	epf_bar[bar].barno = 0;
246 	epf_bar[bar].flags = 0;
247 }
248 EXPORT_SYMBOL_GPL(pci_epf_free_space);
249 
250 /**
251  * pci_epf_alloc_space() - allocate memory for the PCI EPF register space
252  * @epf: the EPF device to whom allocate the memory
253  * @size: the size of the memory that has to be allocated
254  * @bar: the BAR number corresponding to the allocated register space
255  * @epc_features: the features provided by the EPC specific to this EPF
256  * @type: Identifies if the allocation is for primary EPC or secondary EPC
257  *
258  * Invoke to allocate memory for the PCI EPF register space.
259  * Flag PCI_BASE_ADDRESS_MEM_TYPE_64 will automatically get set if the BAR
260  * can only be a 64-bit BAR, or if the requested size is larger than 2 GB.
261  */
pci_epf_alloc_space(struct pci_epf * epf,size_t size,enum pci_barno bar,const struct pci_epc_features * epc_features,enum pci_epc_interface_type type)262 void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
263 			  const struct pci_epc_features *epc_features,
264 			  enum pci_epc_interface_type type)
265 {
266 	u64 bar_fixed_size = epc_features->bar[bar].fixed_size;
267 	size_t align = epc_features->align;
268 	struct pci_epf_bar *epf_bar;
269 	dma_addr_t phys_addr;
270 	struct pci_epc *epc;
271 	struct device *dev;
272 	void *space;
273 
274 	if (size < 128)
275 		size = 128;
276 
277 	/* According to PCIe base spec, min size for a resizable BAR is 1 MB. */
278 	if (epc_features->bar[bar].type == BAR_RESIZABLE && size < SZ_1M)
279 		size = SZ_1M;
280 
281 	if (epc_features->bar[bar].type == BAR_FIXED && bar_fixed_size) {
282 		if (size > bar_fixed_size) {
283 			dev_err(&epf->dev,
284 				"requested BAR size is larger than fixed size\n");
285 			return NULL;
286 		}
287 		size = bar_fixed_size;
288 	}
289 
290 	if (align)
291 		size = ALIGN(size, align);
292 	else
293 		size = roundup_pow_of_two(size);
294 
295 	if (type == PRIMARY_INTERFACE) {
296 		epc = epf->epc;
297 		epf_bar = epf->bar;
298 	} else {
299 		epc = epf->sec_epc;
300 		epf_bar = epf->sec_epc_bar;
301 	}
302 
303 	dev = epc->dev.parent;
304 	space = dma_alloc_coherent(dev, size, &phys_addr, GFP_KERNEL);
305 	if (!space) {
306 		dev_err(dev, "failed to allocate mem space\n");
307 		return NULL;
308 	}
309 
310 	epf_bar[bar].phys_addr = phys_addr;
311 	epf_bar[bar].addr = space;
312 	epf_bar[bar].size = size;
313 	epf_bar[bar].barno = bar;
314 	if (upper_32_bits(size) || epc_features->bar[bar].only_64bit)
315 		epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;
316 	else
317 		epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_32;
318 
319 	return space;
320 }
321 EXPORT_SYMBOL_GPL(pci_epf_alloc_space);
322 
pci_epf_remove_cfs(struct pci_epf_driver * driver)323 static void pci_epf_remove_cfs(struct pci_epf_driver *driver)
324 {
325 	struct config_group *group, *tmp;
326 
327 	if (!IS_ENABLED(CONFIG_PCI_ENDPOINT_CONFIGFS))
328 		return;
329 
330 	mutex_lock(&pci_epf_mutex);
331 	list_for_each_entry_safe(group, tmp, &driver->epf_group, group_entry)
332 		pci_ep_cfs_remove_epf_group(group);
333 	list_del(&driver->epf_group);
334 	mutex_unlock(&pci_epf_mutex);
335 }
336 
337 /**
338  * pci_epf_unregister_driver() - unregister the PCI EPF driver
339  * @driver: the PCI EPF driver that has to be unregistered
340  *
341  * Invoke to unregister the PCI EPF driver.
342  */
pci_epf_unregister_driver(struct pci_epf_driver * driver)343 void pci_epf_unregister_driver(struct pci_epf_driver *driver)
344 {
345 	pci_epf_remove_cfs(driver);
346 	driver_unregister(&driver->driver);
347 }
348 EXPORT_SYMBOL_GPL(pci_epf_unregister_driver);
349 
pci_epf_add_cfs(struct pci_epf_driver * driver)350 static int pci_epf_add_cfs(struct pci_epf_driver *driver)
351 {
352 	struct config_group *group;
353 	const struct pci_epf_device_id *id;
354 
355 	if (!IS_ENABLED(CONFIG_PCI_ENDPOINT_CONFIGFS))
356 		return 0;
357 
358 	INIT_LIST_HEAD(&driver->epf_group);
359 
360 	id = driver->id_table;
361 	while (id->name[0]) {
362 		group = pci_ep_cfs_add_epf_group(id->name);
363 		if (IS_ERR(group)) {
364 			pci_epf_remove_cfs(driver);
365 			return PTR_ERR(group);
366 		}
367 
368 		mutex_lock(&pci_epf_mutex);
369 		list_add_tail(&group->group_entry, &driver->epf_group);
370 		mutex_unlock(&pci_epf_mutex);
371 		id++;
372 	}
373 
374 	return 0;
375 }
376 
377 /**
378  * __pci_epf_register_driver() - register a new PCI EPF driver
379  * @driver: structure representing PCI EPF driver
380  * @owner: the owner of the module that registers the PCI EPF driver
381  *
382  * Invoke to register a new PCI EPF driver.
383  */
__pci_epf_register_driver(struct pci_epf_driver * driver,struct module * owner)384 int __pci_epf_register_driver(struct pci_epf_driver *driver,
385 			      struct module *owner)
386 {
387 	int ret;
388 
389 	if (!driver->ops)
390 		return -EINVAL;
391 
392 	if (!driver->ops->bind || !driver->ops->unbind)
393 		return -EINVAL;
394 
395 	driver->driver.bus = &pci_epf_bus_type;
396 	driver->driver.owner = owner;
397 
398 	ret = driver_register(&driver->driver);
399 	if (ret)
400 		return ret;
401 
402 	pci_epf_add_cfs(driver);
403 
404 	return 0;
405 }
406 EXPORT_SYMBOL_GPL(__pci_epf_register_driver);
407 
408 /**
409  * pci_epf_destroy() - destroy the created PCI EPF device
410  * @epf: the PCI EPF device that has to be destroyed.
411  *
412  * Invoke to destroy the PCI EPF device created by invoking pci_epf_create().
413  */
pci_epf_destroy(struct pci_epf * epf)414 void pci_epf_destroy(struct pci_epf *epf)
415 {
416 	device_unregister(&epf->dev);
417 }
418 EXPORT_SYMBOL_GPL(pci_epf_destroy);
419 
420 /**
421  * pci_epf_create() - create a new PCI EPF device
422  * @name: the name of the PCI EPF device. This name will be used to bind the
423  *	  EPF device to a EPF driver
424  *
425  * Invoke to create a new PCI EPF device by providing the name of the function
426  * device.
427  */
pci_epf_create(const char * name)428 struct pci_epf *pci_epf_create(const char *name)
429 {
430 	int ret;
431 	struct pci_epf *epf;
432 	struct device *dev;
433 	int len;
434 
435 	epf = kzalloc(sizeof(*epf), GFP_KERNEL);
436 	if (!epf)
437 		return ERR_PTR(-ENOMEM);
438 
439 	len = strchrnul(name, '.') - name;
440 	epf->name = kstrndup(name, len, GFP_KERNEL);
441 	if (!epf->name) {
442 		kfree(epf);
443 		return ERR_PTR(-ENOMEM);
444 	}
445 
446 	/* VFs are numbered starting with 1. So set BIT(0) by default */
447 	epf->vfunction_num_map = 1;
448 	INIT_LIST_HEAD(&epf->pci_vepf);
449 
450 	dev = &epf->dev;
451 	device_initialize(dev);
452 	dev->bus = &pci_epf_bus_type;
453 	dev->type = &pci_epf_type;
454 	mutex_init(&epf->lock);
455 
456 	ret = dev_set_name(dev, "%s", name);
457 	if (ret) {
458 		put_device(dev);
459 		return ERR_PTR(ret);
460 	}
461 
462 	ret = device_add(dev);
463 	if (ret) {
464 		put_device(dev);
465 		return ERR_PTR(ret);
466 	}
467 
468 	return epf;
469 }
470 EXPORT_SYMBOL_GPL(pci_epf_create);
471 
pci_epf_dev_release(struct device * dev)472 static void pci_epf_dev_release(struct device *dev)
473 {
474 	struct pci_epf *epf = to_pci_epf(dev);
475 
476 	kfree(epf->name);
477 	kfree(epf);
478 }
479 
480 static const struct device_type pci_epf_type = {
481 	.release	= pci_epf_dev_release,
482 };
483 
484 static const struct pci_epf_device_id *
pci_epf_match_id(const struct pci_epf_device_id * id,const struct pci_epf * epf)485 pci_epf_match_id(const struct pci_epf_device_id *id, const struct pci_epf *epf)
486 {
487 	while (id->name[0]) {
488 		if (strcmp(epf->name, id->name) == 0)
489 			return id;
490 		id++;
491 	}
492 
493 	return NULL;
494 }
495 
pci_epf_device_match(struct device * dev,const struct device_driver * drv)496 static int pci_epf_device_match(struct device *dev, const struct device_driver *drv)
497 {
498 	struct pci_epf *epf = to_pci_epf(dev);
499 	const struct pci_epf_driver *driver = to_pci_epf_driver(drv);
500 
501 	if (driver->id_table)
502 		return !!pci_epf_match_id(driver->id_table, epf);
503 
504 	return !strcmp(epf->name, drv->name);
505 }
506 
pci_epf_device_probe(struct device * dev)507 static int pci_epf_device_probe(struct device *dev)
508 {
509 	struct pci_epf *epf = to_pci_epf(dev);
510 	struct pci_epf_driver *driver = to_pci_epf_driver(dev->driver);
511 
512 	if (!driver->probe)
513 		return -ENODEV;
514 
515 	epf->driver = driver;
516 
517 	return driver->probe(epf, pci_epf_match_id(driver->id_table, epf));
518 }
519 
pci_epf_device_remove(struct device * dev)520 static void pci_epf_device_remove(struct device *dev)
521 {
522 	struct pci_epf *epf = to_pci_epf(dev);
523 	struct pci_epf_driver *driver = to_pci_epf_driver(dev->driver);
524 
525 	if (driver->remove)
526 		driver->remove(epf);
527 	epf->driver = NULL;
528 }
529 
530 static const struct bus_type pci_epf_bus_type = {
531 	.name		= "pci-epf",
532 	.match		= pci_epf_device_match,
533 	.probe		= pci_epf_device_probe,
534 	.remove		= pci_epf_device_remove,
535 };
536 
pci_epf_init(void)537 static int __init pci_epf_init(void)
538 {
539 	int ret;
540 
541 	ret = bus_register(&pci_epf_bus_type);
542 	if (ret) {
543 		pr_err("failed to register pci epf bus --> %d\n", ret);
544 		return ret;
545 	}
546 
547 	return 0;
548 }
549 module_init(pci_epf_init);
550 
pci_epf_exit(void)551 static void __exit pci_epf_exit(void)
552 {
553 	bus_unregister(&pci_epf_bus_type);
554 }
555 module_exit(pci_epf_exit);
556 
557 MODULE_DESCRIPTION("PCI EPF Library");
558 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
559