1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright IBM Corp. 2020
4  *
5  * Author(s):
6  *   Pierre Morel <pmorel@linux.ibm.com>
7  *
8  */
9 
10 #define KMSG_COMPONENT "zpci"
11 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
12 
13 #include <linux/kernel.h>
14 #include <linux/slab.h>
15 #include <linux/err.h>
16 #include <linux/export.h>
17 #include <linux/delay.h>
18 #include <linux/seq_file.h>
19 #include <linux/jump_label.h>
20 #include <linux/pci.h>
21 #include <linux/printk.h>
22 #include <linux/dma-direct.h>
23 
24 #include <asm/pci_clp.h>
25 #include <asm/pci_dma.h>
26 
27 #include "pci_bus.h"
28 #include "pci_iov.h"
29 
30 static LIST_HEAD(zbus_list);
31 static DEFINE_MUTEX(zbus_list_lock);
32 static int zpci_nb_devices;
33 
34 /* zpci_bus_prepare_device - Prepare a zPCI function for scanning
35  * @zdev: the zPCI function to be prepared
36  *
37  * The PCI resources for the function are set up and added to its zbus and the
38  * function is enabled. The function must be added to a zbus which must have
39  * a PCI bus created. If an error occurs the zPCI function is not enabled.
40  *
41  * Return: 0 on success, an error code otherwise
42  */
43 static int zpci_bus_prepare_device(struct zpci_dev *zdev)
44 {
45 	int rc, i;
46 
47 	if (!zdev_enabled(zdev)) {
48 		rc = zpci_enable_device(zdev);
49 		if (rc)
50 			return rc;
51 	}
52 
53 	if (!zdev->has_resources) {
54 		zpci_setup_bus_resources(zdev);
55 		for (i = 0; i < PCI_STD_NUM_BARS; i++) {
56 			if (zdev->bars[i].res)
57 				pci_bus_add_resource(zdev->zbus->bus, zdev->bars[i].res);
58 		}
59 	}
60 
61 	return 0;
62 }
63 
64 /* zpci_bus_scan_device - Scan a single device adding it to the PCI core
65  * @zdev: the zdev to be scanned
66  *
67  * Scans the PCI function making it available to the common PCI code.
68  *
69  * Return: 0 on success, an error value otherwise
70  */
71 int zpci_bus_scan_device(struct zpci_dev *zdev)
72 {
73 	struct pci_dev *pdev;
74 	int rc;
75 
76 	rc = zpci_bus_prepare_device(zdev);
77 	if (rc)
78 		return rc;
79 
80 	pdev = pci_scan_single_device(zdev->zbus->bus, zdev->devfn);
81 	if (!pdev)
82 		return -ENODEV;
83 
84 	pci_lock_rescan_remove();
85 	pci_bus_add_device(pdev);
86 	pci_unlock_rescan_remove();
87 
88 	return 0;
89 }
90 
91 /* zpci_bus_remove_device - Removes the given zdev from the PCI core
92  * @zdev: the zdev to be removed from the PCI core
93  * @set_error: if true the device's error state is set to permanent failure
94  *
95  * Sets a zPCI device to a configured but offline state; the zPCI
96  * device is still accessible through its hotplug slot and the zPCI
97  * API but is removed from the common code PCI bus, making it
98  * no longer available to drivers.
99  */
100 void zpci_bus_remove_device(struct zpci_dev *zdev, bool set_error)
101 {
102 	struct zpci_bus *zbus = zdev->zbus;
103 	struct pci_dev *pdev;
104 
105 	if (!zdev->zbus->bus)
106 		return;
107 
108 	pdev = pci_get_slot(zbus->bus, zdev->devfn);
109 	if (pdev) {
110 		if (set_error)
111 			pdev->error_state = pci_channel_io_perm_failure;
112 		if (pdev->is_virtfn) {
113 			zpci_iov_remove_virtfn(pdev, zdev->vfn);
114 			/* balance pci_get_slot */
115 			pci_dev_put(pdev);
116 			return;
117 		}
118 		pci_stop_and_remove_bus_device_locked(pdev);
119 		/* balance pci_get_slot */
120 		pci_dev_put(pdev);
121 	}
122 }
123 
124 /* zpci_bus_scan_bus - Scan all configured zPCI functions on the bus
125  * @zbus: the zbus to be scanned
126  *
127  * Enables and scans all PCI functions on the bus making them available to the
128  * common PCI code. If a PCI function fails to be initialized an error will be
129  * returned but attempts will still be made for all other functions on the bus.
130  *
131  * Return: 0 on success, an error value otherwise
132  */
133 int zpci_bus_scan_bus(struct zpci_bus *zbus)
134 {
135 	struct zpci_dev *zdev;
136 	int devfn, rc, ret = 0;
137 
138 	for (devfn = 0; devfn < ZPCI_FUNCTIONS_PER_BUS; devfn++) {
139 		zdev = zbus->function[devfn];
140 		if (zdev && zdev->state == ZPCI_FN_STATE_CONFIGURED) {
141 			rc = zpci_bus_prepare_device(zdev);
142 			if (rc)
143 				ret = -EIO;
144 		}
145 	}
146 
147 	pci_lock_rescan_remove();
148 	pci_scan_child_bus(zbus->bus);
149 	pci_bus_add_devices(zbus->bus);
150 	pci_unlock_rescan_remove();
151 
152 	return ret;
153 }
154 
155 /* zpci_bus_scan_busses - Scan all registered busses
156  *
157  * Scan all available zbusses
158  *
159  */
160 void zpci_bus_scan_busses(void)
161 {
162 	struct zpci_bus *zbus = NULL;
163 
164 	mutex_lock(&zbus_list_lock);
165 	list_for_each_entry(zbus, &zbus_list, bus_next) {
166 		zpci_bus_scan_bus(zbus);
167 		cond_resched();
168 	}
169 	mutex_unlock(&zbus_list_lock);
170 }
171 
172 static bool zpci_bus_is_multifunction_root(struct zpci_dev *zdev)
173 {
174 	return !s390_pci_no_rid && zdev->rid_available &&
175 		!zdev->vfn;
176 }
177 
178 /* zpci_bus_create_pci_bus - Create the PCI bus associated with this zbus
179  * @zbus: the zbus holding the zdevices
180  * @fr: PCI root function that will determine the bus's domain, and bus speed
181  * @ops: the pci operations
182  *
183  * The PCI function @fr determines the domain (its UID), multifunction property
184  * and maximum bus speed of the entire bus.
185  *
186  * Return: 0 on success, an error code otherwise
187  */
188 static int zpci_bus_create_pci_bus(struct zpci_bus *zbus, struct zpci_dev *fr, struct pci_ops *ops)
189 {
190 	struct pci_bus *bus;
191 	int domain;
192 
193 	domain = zpci_alloc_domain((u16)fr->uid);
194 	if (domain < 0)
195 		return domain;
196 
197 	zbus->domain_nr = domain;
198 	zbus->multifunction = zpci_bus_is_multifunction_root(fr);
199 	zbus->max_bus_speed = fr->max_bus_speed;
200 
201 	/*
202 	 * Note that the zbus->resources are taken over and zbus->resources
203 	 * is empty after a successful call
204 	 */
205 	bus = pci_create_root_bus(NULL, ZPCI_BUS_NR, ops, zbus, &zbus->resources);
206 	if (!bus) {
207 		zpci_free_domain(zbus->domain_nr);
208 		return -EFAULT;
209 	}
210 
211 	zbus->bus = bus;
212 
213 	return 0;
214 }
215 
216 static void zpci_bus_release(struct kref *kref)
217 {
218 	struct zpci_bus *zbus = container_of(kref, struct zpci_bus, kref);
219 
220 	if (zbus->bus) {
221 		pci_lock_rescan_remove();
222 		pci_stop_root_bus(zbus->bus);
223 
224 		zpci_free_domain(zbus->domain_nr);
225 		pci_free_resource_list(&zbus->resources);
226 
227 		pci_remove_root_bus(zbus->bus);
228 		pci_unlock_rescan_remove();
229 	}
230 
231 	mutex_lock(&zbus_list_lock);
232 	list_del(&zbus->bus_next);
233 	mutex_unlock(&zbus_list_lock);
234 	kfree(zbus);
235 }
236 
237 static void zpci_bus_put(struct zpci_bus *zbus)
238 {
239 	kref_put(&zbus->kref, zpci_bus_release);
240 }
241 
242 static struct zpci_bus *zpci_bus_get(int topo, bool topo_is_tid)
243 {
244 	struct zpci_bus *zbus;
245 
246 	mutex_lock(&zbus_list_lock);
247 	list_for_each_entry(zbus, &zbus_list, bus_next) {
248 		if (!zbus->multifunction)
249 			continue;
250 		if (topo_is_tid == zbus->topo_is_tid && topo == zbus->topo) {
251 			kref_get(&zbus->kref);
252 			goto out_unlock;
253 		}
254 	}
255 	zbus = NULL;
256 out_unlock:
257 	mutex_unlock(&zbus_list_lock);
258 	return zbus;
259 }
260 
261 static struct zpci_bus *zpci_bus_alloc(int topo, bool topo_is_tid)
262 {
263 	struct zpci_bus *zbus;
264 
265 	zbus = kzalloc(sizeof(*zbus), GFP_KERNEL);
266 	if (!zbus)
267 		return NULL;
268 
269 	zbus->topo = topo;
270 	zbus->topo_is_tid = topo_is_tid;
271 	INIT_LIST_HEAD(&zbus->bus_next);
272 	mutex_lock(&zbus_list_lock);
273 	list_add_tail(&zbus->bus_next, &zbus_list);
274 	mutex_unlock(&zbus_list_lock);
275 
276 	kref_init(&zbus->kref);
277 	INIT_LIST_HEAD(&zbus->resources);
278 
279 	zbus->bus_resource.start = 0;
280 	zbus->bus_resource.end = ZPCI_BUS_NR;
281 	zbus->bus_resource.flags = IORESOURCE_BUS;
282 	pci_add_resource(&zbus->resources, &zbus->bus_resource);
283 
284 	return zbus;
285 }
286 
287 static void pci_dma_range_setup(struct pci_dev *pdev)
288 {
289 	struct zpci_dev *zdev = to_zpci(pdev);
290 	u64 aligned_end, size;
291 	dma_addr_t dma_start;
292 	int ret;
293 
294 	dma_start = PAGE_ALIGN(zdev->start_dma);
295 	aligned_end = PAGE_ALIGN_DOWN(zdev->end_dma + 1);
296 	if (aligned_end >= dma_start)
297 		size = aligned_end - dma_start;
298 	else
299 		size = 0;
300 	WARN_ON_ONCE(size == 0);
301 
302 	ret = dma_direct_set_offset(&pdev->dev, 0, dma_start, size);
303 	if (ret)
304 		pr_err("Failed to allocate DMA range map for %s\n", pci_name(pdev));
305 }
306 
307 void pcibios_bus_add_device(struct pci_dev *pdev)
308 {
309 	struct zpci_dev *zdev = to_zpci(pdev);
310 
311 	pci_dma_range_setup(pdev);
312 
313 	/*
314 	 * With pdev->no_vf_scan the common PCI probing code does not
315 	 * perform PF/VF linking.
316 	 */
317 	if (zdev->vfn) {
318 		zpci_iov_setup_virtfn(zdev->zbus, pdev, zdev->vfn);
319 		pdev->no_command_memory = 1;
320 	}
321 }
322 
323 static int zpci_bus_add_device(struct zpci_bus *zbus, struct zpci_dev *zdev)
324 {
325 	int rc = -EINVAL;
326 
327 	if (zbus->multifunction) {
328 		if (!zdev->rid_available) {
329 			WARN_ONCE(1, "rid_available not set for multifunction\n");
330 			return rc;
331 		}
332 		zdev->devfn = zdev->rid & ZPCI_RID_MASK_DEVFN;
333 	}
334 
335 	if (zbus->function[zdev->devfn]) {
336 		pr_err("devfn %04x is already assigned\n", zdev->devfn);
337 		return rc;
338 	}
339 	zdev->zbus = zbus;
340 	zbus->function[zdev->devfn] = zdev;
341 	zpci_nb_devices++;
342 
343 	rc = zpci_init_slot(zdev);
344 	if (rc)
345 		goto error;
346 	zdev->has_hp_slot = 1;
347 
348 	return 0;
349 
350 error:
351 	zbus->function[zdev->devfn] = NULL;
352 	zdev->zbus = NULL;
353 	zpci_nb_devices--;
354 	return rc;
355 }
356 
357 static bool zpci_bus_is_isolated_vf(struct zpci_bus *zbus, struct zpci_dev *zdev)
358 {
359 	struct pci_dev *pdev;
360 
361 	if (!zdev->vfn)
362 		return false;
363 
364 	pdev = zpci_iov_find_parent_pf(zbus, zdev);
365 	if (!pdev)
366 		return true;
367 	pci_dev_put(pdev);
368 	return false;
369 }
370 
371 int zpci_bus_device_register(struct zpci_dev *zdev, struct pci_ops *ops)
372 {
373 	bool topo_is_tid = zdev->tid_avail;
374 	struct zpci_bus *zbus = NULL;
375 	int topo, rc = -EBADF;
376 
377 	if (zpci_nb_devices == ZPCI_NR_DEVICES) {
378 		pr_warn("Adding PCI function %08x failed because the configured limit of %d is reached\n",
379 			zdev->fid, ZPCI_NR_DEVICES);
380 		return -ENOSPC;
381 	}
382 
383 	topo = topo_is_tid ? zdev->tid : zdev->pchid;
384 	zbus = zpci_bus_get(topo, topo_is_tid);
385 	/*
386 	 * An isolated VF gets its own domain/bus even if there exists
387 	 * a matching domain/bus already
388 	 */
389 	if (zbus && zpci_bus_is_isolated_vf(zbus, zdev)) {
390 		zpci_bus_put(zbus);
391 		zbus = NULL;
392 	}
393 
394 	if (!zbus) {
395 		zbus = zpci_bus_alloc(topo, topo_is_tid);
396 		if (!zbus)
397 			return -ENOMEM;
398 	}
399 
400 	if (!zbus->bus) {
401 		/* The UID of the first PCI function registered with a zpci_bus
402 		 * is used as the domain number for that bus. Currently there
403 		 * is exactly one zpci_bus per domain.
404 		 */
405 		rc = zpci_bus_create_pci_bus(zbus, zdev, ops);
406 		if (rc)
407 			goto error;
408 	}
409 
410 	rc = zpci_bus_add_device(zbus, zdev);
411 	if (rc)
412 		goto error;
413 
414 	return 0;
415 
416 error:
417 	pr_err("Adding PCI function %08x failed\n", zdev->fid);
418 	zpci_bus_put(zbus);
419 	return rc;
420 }
421 
422 void zpci_bus_device_unregister(struct zpci_dev *zdev)
423 {
424 	struct zpci_bus *zbus = zdev->zbus;
425 
426 	zpci_nb_devices--;
427 	zbus->function[zdev->devfn] = NULL;
428 	zpci_bus_put(zbus);
429 }
430