1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Implement the AER root port service driver. The driver registers an IRQ
4 * handler. When a root port triggers an AER interrupt, the IRQ handler
5 * collects Root Port status and schedules work.
6 *
7 * Copyright (C) 2006 Intel Corp.
8 * Tom Long Nguyen (tom.l.nguyen@intel.com)
9 * Zhang Yanmin (yanmin.zhang@intel.com)
10 *
11 * (C) Copyright 2009 Hewlett-Packard Development Company, L.P.
12 * Andrew Patterson <andrew.patterson@hp.com>
13 */
14
15 #define pr_fmt(fmt) "AER: " fmt
16 #define dev_fmt pr_fmt
17
18 #include <linux/bitops.h>
19 #include <linux/cper.h>
20 #include <linux/dev_printk.h>
21 #include <linux/pci.h>
22 #include <linux/pci-acpi.h>
23 #include <linux/sched.h>
24 #include <linux/kernel.h>
25 #include <linux/errno.h>
26 #include <linux/pm.h>
27 #include <linux/init.h>
28 #include <linux/interrupt.h>
29 #include <linux/delay.h>
30 #include <linux/kfifo.h>
31 #include <linux/slab.h>
32 #include <acpi/apei.h>
33 #include <acpi/ghes.h>
34 #include <ras/ras_event.h>
35
36 #include "../pci.h"
37 #include "portdrv.h"
38
39 #define aer_printk(level, pdev, fmt, arg...) \
40 dev_printk(level, &(pdev)->dev, fmt, ##arg)
41
42 #define AER_ERROR_SOURCES_MAX 128
43
44 #define AER_MAX_TYPEOF_COR_ERRS 16 /* as per PCI_ERR_COR_STATUS */
45 #define AER_MAX_TYPEOF_UNCOR_ERRS 27 /* as per PCI_ERR_UNCOR_STATUS*/
46
47 struct aer_err_source {
48 u32 status; /* PCI_ERR_ROOT_STATUS */
49 u32 id; /* PCI_ERR_ROOT_ERR_SRC */
50 };
51
52 struct aer_rpc {
53 struct pci_dev *rpd; /* Root Port device */
54 DECLARE_KFIFO(aer_fifo, struct aer_err_source, AER_ERROR_SOURCES_MAX);
55 };
56
57 /* AER stats for the device */
58 struct aer_stats {
59
60 /*
61 * Fields for all AER capable devices. They indicate the errors
62 * "as seen by this device". Note that this may mean that if an
63 * Endpoint is causing problems, the AER counters may increment
64 * at its link partner (e.g. Root Port) because the errors will be
65 * "seen" by the link partner and not the problematic Endpoint
66 * itself (which may report all counters as 0 as it never saw any
67 * problems).
68 */
69 /* Counters for different type of correctable errors */
70 u64 dev_cor_errs[AER_MAX_TYPEOF_COR_ERRS];
71 /* Counters for different type of fatal uncorrectable errors */
72 u64 dev_fatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS];
73 /* Counters for different type of nonfatal uncorrectable errors */
74 u64 dev_nonfatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS];
75 /* Total number of ERR_COR sent by this device */
76 u64 dev_total_cor_errs;
77 /* Total number of ERR_FATAL sent by this device */
78 u64 dev_total_fatal_errs;
79 /* Total number of ERR_NONFATAL sent by this device */
80 u64 dev_total_nonfatal_errs;
81
82 /*
83 * Fields for Root Ports & Root Complex Event Collectors only; these
84 * indicate the total number of ERR_COR, ERR_FATAL, and ERR_NONFATAL
85 * messages received by the Root Port / Event Collector, INCLUDING the
86 * ones that are generated internally (by the Root Port itself)
87 */
88 u64 rootport_total_cor_errs;
89 u64 rootport_total_fatal_errs;
90 u64 rootport_total_nonfatal_errs;
91 };
92
93 #define AER_LOG_TLP_MASKS (PCI_ERR_UNC_POISON_TLP| \
94 PCI_ERR_UNC_ECRC| \
95 PCI_ERR_UNC_UNSUP| \
96 PCI_ERR_UNC_COMP_ABORT| \
97 PCI_ERR_UNC_UNX_COMP| \
98 PCI_ERR_UNC_MALF_TLP)
99
100 #define SYSTEM_ERROR_INTR_ON_MESG_MASK (PCI_EXP_RTCTL_SECEE| \
101 PCI_EXP_RTCTL_SENFEE| \
102 PCI_EXP_RTCTL_SEFEE)
103 #define ROOT_PORT_INTR_ON_MESG_MASK (PCI_ERR_ROOT_CMD_COR_EN| \
104 PCI_ERR_ROOT_CMD_NONFATAL_EN| \
105 PCI_ERR_ROOT_CMD_FATAL_EN)
106 #define ERR_COR_ID(d) (d & 0xffff)
107 #define ERR_UNCOR_ID(d) (d >> 16)
108
109 #define AER_ERR_STATUS_MASK (PCI_ERR_ROOT_UNCOR_RCV | \
110 PCI_ERR_ROOT_COR_RCV | \
111 PCI_ERR_ROOT_MULTI_COR_RCV | \
112 PCI_ERR_ROOT_MULTI_UNCOR_RCV)
113
114 static int pcie_aer_disable;
115 static pci_ers_result_t aer_root_reset(struct pci_dev *dev);
116
pci_no_aer(void)117 void pci_no_aer(void)
118 {
119 pcie_aer_disable = 1;
120 }
121
pci_aer_available(void)122 bool pci_aer_available(void)
123 {
124 return !pcie_aer_disable && pci_msi_enabled();
125 }
126
127 #ifdef CONFIG_PCIE_ECRC
128
129 #define ECRC_POLICY_DEFAULT 0 /* ECRC set by BIOS */
130 #define ECRC_POLICY_OFF 1 /* ECRC off for performance */
131 #define ECRC_POLICY_ON 2 /* ECRC on for data integrity */
132
133 static int ecrc_policy = ECRC_POLICY_DEFAULT;
134
135 static const char * const ecrc_policy_str[] = {
136 [ECRC_POLICY_DEFAULT] = "bios",
137 [ECRC_POLICY_OFF] = "off",
138 [ECRC_POLICY_ON] = "on"
139 };
140
141 /**
142 * enable_ecrc_checking - enable PCIe ECRC checking for a device
143 * @dev: the PCI device
144 *
145 * Return: 0 on success, or negative on failure.
146 */
enable_ecrc_checking(struct pci_dev * dev)147 static int enable_ecrc_checking(struct pci_dev *dev)
148 {
149 int aer = dev->aer_cap;
150 u32 reg32;
151
152 if (!aer)
153 return -ENODEV;
154
155 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32);
156 if (reg32 & PCI_ERR_CAP_ECRC_GENC)
157 reg32 |= PCI_ERR_CAP_ECRC_GENE;
158 if (reg32 & PCI_ERR_CAP_ECRC_CHKC)
159 reg32 |= PCI_ERR_CAP_ECRC_CHKE;
160 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32);
161
162 return 0;
163 }
164
165 /**
166 * disable_ecrc_checking - disable PCIe ECRC checking for a device
167 * @dev: the PCI device
168 *
169 * Return: 0 on success, or negative on failure.
170 */
disable_ecrc_checking(struct pci_dev * dev)171 static int disable_ecrc_checking(struct pci_dev *dev)
172 {
173 int aer = dev->aer_cap;
174 u32 reg32;
175
176 if (!aer)
177 return -ENODEV;
178
179 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32);
180 reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE);
181 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32);
182
183 return 0;
184 }
185
186 /**
187 * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based
188 * on global policy
189 * @dev: the PCI device
190 */
pcie_set_ecrc_checking(struct pci_dev * dev)191 void pcie_set_ecrc_checking(struct pci_dev *dev)
192 {
193 if (!pcie_aer_is_native(dev))
194 return;
195
196 switch (ecrc_policy) {
197 case ECRC_POLICY_DEFAULT:
198 return;
199 case ECRC_POLICY_OFF:
200 disable_ecrc_checking(dev);
201 break;
202 case ECRC_POLICY_ON:
203 enable_ecrc_checking(dev);
204 break;
205 default:
206 return;
207 }
208 }
209
210 /**
211 * pcie_ecrc_get_policy - parse kernel command-line ecrc option
212 * @str: ECRC policy from kernel command line to use
213 */
pcie_ecrc_get_policy(char * str)214 void pcie_ecrc_get_policy(char *str)
215 {
216 int i;
217
218 i = match_string(ecrc_policy_str, ARRAY_SIZE(ecrc_policy_str), str);
219 if (i < 0)
220 return;
221
222 ecrc_policy = i;
223 }
224 #endif /* CONFIG_PCIE_ECRC */
225
226 #define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \
227 PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE)
228
pcie_aer_is_native(struct pci_dev * dev)229 int pcie_aer_is_native(struct pci_dev *dev)
230 {
231 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
232
233 if (!dev->aer_cap)
234 return 0;
235
236 return pcie_ports_native || host->native_aer;
237 }
238 EXPORT_SYMBOL_NS_GPL(pcie_aer_is_native, "CXL");
239
pci_enable_pcie_error_reporting(struct pci_dev * dev)240 static int pci_enable_pcie_error_reporting(struct pci_dev *dev)
241 {
242 int rc;
243
244 if (!pcie_aer_is_native(dev))
245 return -EIO;
246
247 rc = pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS);
248 return pcibios_err_to_errno(rc);
249 }
250
pci_aer_clear_nonfatal_status(struct pci_dev * dev)251 int pci_aer_clear_nonfatal_status(struct pci_dev *dev)
252 {
253 int aer = dev->aer_cap;
254 u32 status, sev;
255
256 if (!pcie_aer_is_native(dev))
257 return -EIO;
258
259 /* Clear status bits for ERR_NONFATAL errors only */
260 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
261 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev);
262 status &= ~sev;
263 if (status)
264 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);
265
266 return 0;
267 }
268 EXPORT_SYMBOL_GPL(pci_aer_clear_nonfatal_status);
269
pci_aer_clear_fatal_status(struct pci_dev * dev)270 void pci_aer_clear_fatal_status(struct pci_dev *dev)
271 {
272 int aer = dev->aer_cap;
273 u32 status, sev;
274
275 if (!pcie_aer_is_native(dev))
276 return;
277
278 /* Clear status bits for ERR_FATAL errors only */
279 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
280 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev);
281 status &= sev;
282 if (status)
283 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);
284 }
285
286 /**
287 * pci_aer_raw_clear_status - Clear AER error registers.
288 * @dev: the PCI device
289 *
290 * Clear AER error status registers unconditionally, regardless of
291 * whether they're owned by firmware or the OS.
292 *
293 * Return: 0 on success, or negative on failure.
294 */
pci_aer_raw_clear_status(struct pci_dev * dev)295 int pci_aer_raw_clear_status(struct pci_dev *dev)
296 {
297 int aer = dev->aer_cap;
298 u32 status;
299 int port_type;
300
301 if (!aer)
302 return -EIO;
303
304 port_type = pci_pcie_type(dev);
305 if (port_type == PCI_EXP_TYPE_ROOT_PORT ||
306 port_type == PCI_EXP_TYPE_RC_EC) {
307 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, &status);
308 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, status);
309 }
310
311 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status);
312 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, status);
313
314 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
315 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);
316
317 return 0;
318 }
319
pci_aer_clear_status(struct pci_dev * dev)320 int pci_aer_clear_status(struct pci_dev *dev)
321 {
322 if (!pcie_aer_is_native(dev))
323 return -EIO;
324
325 return pci_aer_raw_clear_status(dev);
326 }
327
pci_save_aer_state(struct pci_dev * dev)328 void pci_save_aer_state(struct pci_dev *dev)
329 {
330 int aer = dev->aer_cap;
331 struct pci_cap_saved_state *save_state;
332 u32 *cap;
333
334 if (!aer)
335 return;
336
337 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR);
338 if (!save_state)
339 return;
340
341 cap = &save_state->cap.data[0];
342 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, cap++);
343 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, cap++);
344 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, cap++);
345 pci_read_config_dword(dev, aer + PCI_ERR_CAP, cap++);
346 if (pcie_cap_has_rtctl(dev))
347 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, cap++);
348 }
349
pci_restore_aer_state(struct pci_dev * dev)350 void pci_restore_aer_state(struct pci_dev *dev)
351 {
352 int aer = dev->aer_cap;
353 struct pci_cap_saved_state *save_state;
354 u32 *cap;
355
356 if (!aer)
357 return;
358
359 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR);
360 if (!save_state)
361 return;
362
363 cap = &save_state->cap.data[0];
364 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, *cap++);
365 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, *cap++);
366 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, *cap++);
367 pci_write_config_dword(dev, aer + PCI_ERR_CAP, *cap++);
368 if (pcie_cap_has_rtctl(dev))
369 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, *cap++);
370 }
371
pci_aer_init(struct pci_dev * dev)372 void pci_aer_init(struct pci_dev *dev)
373 {
374 int n;
375
376 dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
377 if (!dev->aer_cap)
378 return;
379
380 dev->aer_stats = kzalloc(sizeof(struct aer_stats), GFP_KERNEL);
381
382 /*
383 * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER,
384 * PCI_ERR_COR_MASK, and PCI_ERR_CAP. Root and Root Complex Event
385 * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r6.0, sec
386 * 7.8.4.9).
387 */
388 n = pcie_cap_has_rtctl(dev) ? 5 : 4;
389 pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_ERR, sizeof(u32) * n);
390
391 pci_aer_clear_status(dev);
392
393 if (pci_aer_available())
394 pci_enable_pcie_error_reporting(dev);
395
396 pcie_set_ecrc_checking(dev);
397 }
398
pci_aer_exit(struct pci_dev * dev)399 void pci_aer_exit(struct pci_dev *dev)
400 {
401 kfree(dev->aer_stats);
402 dev->aer_stats = NULL;
403 }
404
405 #define AER_AGENT_RECEIVER 0
406 #define AER_AGENT_REQUESTER 1
407 #define AER_AGENT_COMPLETER 2
408 #define AER_AGENT_TRANSMITTER 3
409
410 #define AER_AGENT_REQUESTER_MASK(t) ((t == AER_CORRECTABLE) ? \
411 0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP))
412 #define AER_AGENT_COMPLETER_MASK(t) ((t == AER_CORRECTABLE) ? \
413 0 : PCI_ERR_UNC_COMP_ABORT)
414 #define AER_AGENT_TRANSMITTER_MASK(t) ((t == AER_CORRECTABLE) ? \
415 (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0)
416
417 #define AER_GET_AGENT(t, e) \
418 ((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER : \
419 (e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER : \
420 (e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER : \
421 AER_AGENT_RECEIVER)
422
423 #define AER_PHYSICAL_LAYER_ERROR 0
424 #define AER_DATA_LINK_LAYER_ERROR 1
425 #define AER_TRANSACTION_LAYER_ERROR 2
426
427 #define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \
428 PCI_ERR_COR_RCVR : 0)
429 #define AER_DATA_LINK_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \
430 (PCI_ERR_COR_BAD_TLP| \
431 PCI_ERR_COR_BAD_DLLP| \
432 PCI_ERR_COR_REP_ROLL| \
433 PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP)
434
435 #define AER_GET_LAYER_ERROR(t, e) \
436 ((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \
437 (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \
438 AER_TRANSACTION_LAYER_ERROR)
439
440 /*
441 * AER error strings
442 */
443 static const char * const aer_error_severity_string[] = {
444 "Uncorrectable (Non-Fatal)",
445 "Uncorrectable (Fatal)",
446 "Correctable"
447 };
448
449 static const char *aer_error_layer[] = {
450 "Physical Layer",
451 "Data Link Layer",
452 "Transaction Layer"
453 };
454
455 static const char *aer_correctable_error_string[] = {
456 "RxErr", /* Bit Position 0 */
457 NULL,
458 NULL,
459 NULL,
460 NULL,
461 NULL,
462 "BadTLP", /* Bit Position 6 */
463 "BadDLLP", /* Bit Position 7 */
464 "Rollover", /* Bit Position 8 */
465 NULL,
466 NULL,
467 NULL,
468 "Timeout", /* Bit Position 12 */
469 "NonFatalErr", /* Bit Position 13 */
470 "CorrIntErr", /* Bit Position 14 */
471 "HeaderOF", /* Bit Position 15 */
472 NULL, /* Bit Position 16 */
473 NULL, /* Bit Position 17 */
474 NULL, /* Bit Position 18 */
475 NULL, /* Bit Position 19 */
476 NULL, /* Bit Position 20 */
477 NULL, /* Bit Position 21 */
478 NULL, /* Bit Position 22 */
479 NULL, /* Bit Position 23 */
480 NULL, /* Bit Position 24 */
481 NULL, /* Bit Position 25 */
482 NULL, /* Bit Position 26 */
483 NULL, /* Bit Position 27 */
484 NULL, /* Bit Position 28 */
485 NULL, /* Bit Position 29 */
486 NULL, /* Bit Position 30 */
487 NULL, /* Bit Position 31 */
488 };
489
490 static const char *aer_uncorrectable_error_string[] = {
491 "Undefined", /* Bit Position 0 */
492 NULL,
493 NULL,
494 NULL,
495 "DLP", /* Bit Position 4 */
496 "SDES", /* Bit Position 5 */
497 NULL,
498 NULL,
499 NULL,
500 NULL,
501 NULL,
502 NULL,
503 "TLP", /* Bit Position 12 */
504 "FCP", /* Bit Position 13 */
505 "CmpltTO", /* Bit Position 14 */
506 "CmpltAbrt", /* Bit Position 15 */
507 "UnxCmplt", /* Bit Position 16 */
508 "RxOF", /* Bit Position 17 */
509 "MalfTLP", /* Bit Position 18 */
510 "ECRC", /* Bit Position 19 */
511 "UnsupReq", /* Bit Position 20 */
512 "ACSViol", /* Bit Position 21 */
513 "UncorrIntErr", /* Bit Position 22 */
514 "BlockedTLP", /* Bit Position 23 */
515 "AtomicOpBlocked", /* Bit Position 24 */
516 "TLPBlockedErr", /* Bit Position 25 */
517 "PoisonTLPBlocked", /* Bit Position 26 */
518 NULL, /* Bit Position 27 */
519 NULL, /* Bit Position 28 */
520 NULL, /* Bit Position 29 */
521 NULL, /* Bit Position 30 */
522 NULL, /* Bit Position 31 */
523 };
524
525 static const char *aer_agent_string[] = {
526 "Receiver ID",
527 "Requester ID",
528 "Completer ID",
529 "Transmitter ID"
530 };
531
532 #define aer_stats_dev_attr(name, stats_array, strings_array, \
533 total_string, total_field) \
534 static ssize_t \
535 name##_show(struct device *dev, struct device_attribute *attr, \
536 char *buf) \
537 { \
538 unsigned int i; \
539 struct pci_dev *pdev = to_pci_dev(dev); \
540 u64 *stats = pdev->aer_stats->stats_array; \
541 size_t len = 0; \
542 \
543 for (i = 0; i < ARRAY_SIZE(pdev->aer_stats->stats_array); i++) {\
544 if (strings_array[i]) \
545 len += sysfs_emit_at(buf, len, "%s %llu\n", \
546 strings_array[i], \
547 stats[i]); \
548 else if (stats[i]) \
549 len += sysfs_emit_at(buf, len, \
550 #stats_array "_bit[%d] %llu\n",\
551 i, stats[i]); \
552 } \
553 len += sysfs_emit_at(buf, len, "TOTAL_%s %llu\n", total_string, \
554 pdev->aer_stats->total_field); \
555 return len; \
556 } \
557 static DEVICE_ATTR_RO(name)
558
559 aer_stats_dev_attr(aer_dev_correctable, dev_cor_errs,
560 aer_correctable_error_string, "ERR_COR",
561 dev_total_cor_errs);
562 aer_stats_dev_attr(aer_dev_fatal, dev_fatal_errs,
563 aer_uncorrectable_error_string, "ERR_FATAL",
564 dev_total_fatal_errs);
565 aer_stats_dev_attr(aer_dev_nonfatal, dev_nonfatal_errs,
566 aer_uncorrectable_error_string, "ERR_NONFATAL",
567 dev_total_nonfatal_errs);
568
569 #define aer_stats_rootport_attr(name, field) \
570 static ssize_t \
571 name##_show(struct device *dev, struct device_attribute *attr, \
572 char *buf) \
573 { \
574 struct pci_dev *pdev = to_pci_dev(dev); \
575 return sysfs_emit(buf, "%llu\n", pdev->aer_stats->field); \
576 } \
577 static DEVICE_ATTR_RO(name)
578
579 aer_stats_rootport_attr(aer_rootport_total_err_cor,
580 rootport_total_cor_errs);
581 aer_stats_rootport_attr(aer_rootport_total_err_fatal,
582 rootport_total_fatal_errs);
583 aer_stats_rootport_attr(aer_rootport_total_err_nonfatal,
584 rootport_total_nonfatal_errs);
585
586 static struct attribute *aer_stats_attrs[] __ro_after_init = {
587 &dev_attr_aer_dev_correctable.attr,
588 &dev_attr_aer_dev_fatal.attr,
589 &dev_attr_aer_dev_nonfatal.attr,
590 &dev_attr_aer_rootport_total_err_cor.attr,
591 &dev_attr_aer_rootport_total_err_fatal.attr,
592 &dev_attr_aer_rootport_total_err_nonfatal.attr,
593 NULL
594 };
595
aer_stats_attrs_are_visible(struct kobject * kobj,struct attribute * a,int n)596 static umode_t aer_stats_attrs_are_visible(struct kobject *kobj,
597 struct attribute *a, int n)
598 {
599 struct device *dev = kobj_to_dev(kobj);
600 struct pci_dev *pdev = to_pci_dev(dev);
601
602 if (!pdev->aer_stats)
603 return 0;
604
605 if ((a == &dev_attr_aer_rootport_total_err_cor.attr ||
606 a == &dev_attr_aer_rootport_total_err_fatal.attr ||
607 a == &dev_attr_aer_rootport_total_err_nonfatal.attr) &&
608 ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) &&
609 (pci_pcie_type(pdev) != PCI_EXP_TYPE_RC_EC)))
610 return 0;
611
612 return a->mode;
613 }
614
615 const struct attribute_group aer_stats_attr_group = {
616 .attrs = aer_stats_attrs,
617 .is_visible = aer_stats_attrs_are_visible,
618 };
619
pci_dev_aer_stats_incr(struct pci_dev * pdev,struct aer_err_info * info)620 static void pci_dev_aer_stats_incr(struct pci_dev *pdev,
621 struct aer_err_info *info)
622 {
623 unsigned long status = info->status & ~info->mask;
624 int i, max = -1;
625 u64 *counter = NULL;
626 struct aer_stats *aer_stats = pdev->aer_stats;
627
628 if (!aer_stats)
629 return;
630
631 switch (info->severity) {
632 case AER_CORRECTABLE:
633 aer_stats->dev_total_cor_errs++;
634 counter = &aer_stats->dev_cor_errs[0];
635 max = AER_MAX_TYPEOF_COR_ERRS;
636 break;
637 case AER_NONFATAL:
638 aer_stats->dev_total_nonfatal_errs++;
639 counter = &aer_stats->dev_nonfatal_errs[0];
640 max = AER_MAX_TYPEOF_UNCOR_ERRS;
641 break;
642 case AER_FATAL:
643 aer_stats->dev_total_fatal_errs++;
644 counter = &aer_stats->dev_fatal_errs[0];
645 max = AER_MAX_TYPEOF_UNCOR_ERRS;
646 break;
647 }
648
649 for_each_set_bit(i, &status, max)
650 counter[i]++;
651 }
652
pci_rootport_aer_stats_incr(struct pci_dev * pdev,struct aer_err_source * e_src)653 static void pci_rootport_aer_stats_incr(struct pci_dev *pdev,
654 struct aer_err_source *e_src)
655 {
656 struct aer_stats *aer_stats = pdev->aer_stats;
657
658 if (!aer_stats)
659 return;
660
661 if (e_src->status & PCI_ERR_ROOT_COR_RCV)
662 aer_stats->rootport_total_cor_errs++;
663
664 if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) {
665 if (e_src->status & PCI_ERR_ROOT_FATAL_RCV)
666 aer_stats->rootport_total_fatal_errs++;
667 else
668 aer_stats->rootport_total_nonfatal_errs++;
669 }
670 }
671
__aer_print_error(struct pci_dev * dev,struct aer_err_info * info)672 static void __aer_print_error(struct pci_dev *dev,
673 struct aer_err_info *info)
674 {
675 const char **strings;
676 unsigned long status = info->status & ~info->mask;
677 const char *level, *errmsg;
678 int i;
679
680 if (info->severity == AER_CORRECTABLE) {
681 strings = aer_correctable_error_string;
682 level = KERN_WARNING;
683 } else {
684 strings = aer_uncorrectable_error_string;
685 level = KERN_ERR;
686 }
687
688 for_each_set_bit(i, &status, 32) {
689 errmsg = strings[i];
690 if (!errmsg)
691 errmsg = "Unknown Error Bit";
692
693 aer_printk(level, dev, " [%2d] %-22s%s\n", i, errmsg,
694 info->first_error == i ? " (First)" : "");
695 }
696 pci_dev_aer_stats_incr(dev, info);
697 }
698
aer_print_error(struct pci_dev * dev,struct aer_err_info * info)699 void aer_print_error(struct pci_dev *dev, struct aer_err_info *info)
700 {
701 int layer, agent;
702 int id = pci_dev_id(dev);
703 const char *level;
704
705 if (!info->status) {
706 pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n",
707 aer_error_severity_string[info->severity]);
708 goto out;
709 }
710
711 layer = AER_GET_LAYER_ERROR(info->severity, info->status);
712 agent = AER_GET_AGENT(info->severity, info->status);
713
714 level = (info->severity == AER_CORRECTABLE) ? KERN_WARNING : KERN_ERR;
715
716 aer_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n",
717 aer_error_severity_string[info->severity],
718 aer_error_layer[layer], aer_agent_string[agent]);
719
720 aer_printk(level, dev, " device [%04x:%04x] error status/mask=%08x/%08x\n",
721 dev->vendor, dev->device, info->status, info->mask);
722
723 __aer_print_error(dev, info);
724
725 if (info->tlp_header_valid)
726 pcie_print_tlp_log(dev, &info->tlp, dev_fmt(" "));
727
728 out:
729 if (info->id && info->error_dev_num > 1 && info->id == id)
730 pci_err(dev, " Error of this Agent is reported first\n");
731
732 trace_aer_event(dev_name(&dev->dev), (info->status & ~info->mask),
733 info->severity, info->tlp_header_valid, &info->tlp);
734 }
735
aer_print_port_info(struct pci_dev * dev,struct aer_err_info * info)736 static void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info)
737 {
738 u8 bus = info->id >> 8;
739 u8 devfn = info->id & 0xff;
740
741 pci_info(dev, "%s%s error message received from %04x:%02x:%02x.%d\n",
742 info->multi_error_valid ? "Multiple " : "",
743 aer_error_severity_string[info->severity],
744 pci_domain_nr(dev->bus), bus, PCI_SLOT(devfn),
745 PCI_FUNC(devfn));
746 }
747
748 #ifdef CONFIG_ACPI_APEI_PCIEAER
cper_severity_to_aer(int cper_severity)749 int cper_severity_to_aer(int cper_severity)
750 {
751 switch (cper_severity) {
752 case CPER_SEV_RECOVERABLE:
753 return AER_NONFATAL;
754 case CPER_SEV_FATAL:
755 return AER_FATAL;
756 default:
757 return AER_CORRECTABLE;
758 }
759 }
760 EXPORT_SYMBOL_GPL(cper_severity_to_aer);
761 #endif
762
pci_print_aer(struct pci_dev * dev,int aer_severity,struct aer_capability_regs * aer)763 void pci_print_aer(struct pci_dev *dev, int aer_severity,
764 struct aer_capability_regs *aer)
765 {
766 int layer, agent, tlp_header_valid = 0;
767 u32 status, mask;
768 struct aer_err_info info;
769
770 if (aer_severity == AER_CORRECTABLE) {
771 status = aer->cor_status;
772 mask = aer->cor_mask;
773 } else {
774 status = aer->uncor_status;
775 mask = aer->uncor_mask;
776 tlp_header_valid = status & AER_LOG_TLP_MASKS;
777 }
778
779 layer = AER_GET_LAYER_ERROR(aer_severity, status);
780 agent = AER_GET_AGENT(aer_severity, status);
781
782 memset(&info, 0, sizeof(info));
783 info.severity = aer_severity;
784 info.status = status;
785 info.mask = mask;
786 info.first_error = PCI_ERR_CAP_FEP(aer->cap_control);
787
788 pci_err(dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n", status, mask);
789 __aer_print_error(dev, &info);
790 pci_err(dev, "aer_layer=%s, aer_agent=%s\n",
791 aer_error_layer[layer], aer_agent_string[agent]);
792
793 if (aer_severity != AER_CORRECTABLE)
794 pci_err(dev, "aer_uncor_severity: 0x%08x\n",
795 aer->uncor_severity);
796
797 if (tlp_header_valid)
798 pcie_print_tlp_log(dev, &aer->header_log, dev_fmt(" "));
799
800 trace_aer_event(dev_name(&dev->dev), (status & ~mask),
801 aer_severity, tlp_header_valid, &aer->header_log);
802 }
803 EXPORT_SYMBOL_NS_GPL(pci_print_aer, "CXL");
804
805 /**
806 * add_error_device - list device to be handled
807 * @e_info: pointer to error info
808 * @dev: pointer to pci_dev to be added
809 */
add_error_device(struct aer_err_info * e_info,struct pci_dev * dev)810 static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
811 {
812 if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) {
813 e_info->dev[e_info->error_dev_num] = pci_dev_get(dev);
814 e_info->error_dev_num++;
815 return 0;
816 }
817 return -ENOSPC;
818 }
819
820 /**
821 * is_error_source - check whether the device is source of reported error
822 * @dev: pointer to pci_dev to be checked
823 * @e_info: pointer to reported error info
824 */
is_error_source(struct pci_dev * dev,struct aer_err_info * e_info)825 static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info)
826 {
827 int aer = dev->aer_cap;
828 u32 status, mask;
829 u16 reg16;
830
831 /*
832 * When bus ID is equal to 0, it might be a bad ID
833 * reported by Root Port.
834 */
835 if ((PCI_BUS_NUM(e_info->id) != 0) &&
836 !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) {
837 /* Device ID match? */
838 if (e_info->id == pci_dev_id(dev))
839 return true;
840
841 /* Continue ID comparing if there is no multiple error */
842 if (!e_info->multi_error_valid)
843 return false;
844 }
845
846 /*
847 * When either
848 * 1) bus ID is equal to 0. Some ports might lose the bus
849 * ID of error source id;
850 * 2) bus flag PCI_BUS_FLAGS_NO_AERSID is set
851 * 3) There are multiple errors and prior ID comparing fails;
852 * We check AER status registers to find possible reporter.
853 */
854 if (atomic_read(&dev->enable_cnt) == 0)
855 return false;
856
857 /* Check if AER is enabled */
858 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, ®16);
859 if (!(reg16 & PCI_EXP_AER_FLAGS))
860 return false;
861
862 if (!aer)
863 return false;
864
865 /* Check if error is recorded */
866 if (e_info->severity == AER_CORRECTABLE) {
867 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status);
868 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask);
869 } else {
870 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
871 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask);
872 }
873 if (status & ~mask)
874 return true;
875
876 return false;
877 }
878
find_device_iter(struct pci_dev * dev,void * data)879 static int find_device_iter(struct pci_dev *dev, void *data)
880 {
881 struct aer_err_info *e_info = (struct aer_err_info *)data;
882
883 if (is_error_source(dev, e_info)) {
884 /* List this device */
885 if (add_error_device(e_info, dev)) {
886 /* We cannot handle more... Stop iteration */
887 /* TODO: Should print error message here? */
888 return 1;
889 }
890
891 /* If there is only a single error, stop iteration */
892 if (!e_info->multi_error_valid)
893 return 1;
894 }
895 return 0;
896 }
897
898 /**
899 * find_source_device - search through device hierarchy for source device
900 * @parent: pointer to Root Port pci_dev data structure
901 * @e_info: including detailed error information such as ID
902 *
903 * Return: true if found.
904 *
905 * Invoked by DPC when error is detected at the Root Port.
906 * Caller of this function must set id, severity, and multi_error_valid of
907 * struct aer_err_info pointed by @e_info properly. This function must fill
908 * e_info->error_dev_num and e_info->dev[], based on the given information.
909 */
find_source_device(struct pci_dev * parent,struct aer_err_info * e_info)910 static bool find_source_device(struct pci_dev *parent,
911 struct aer_err_info *e_info)
912 {
913 struct pci_dev *dev = parent;
914 int result;
915
916 /* Must reset in this function */
917 e_info->error_dev_num = 0;
918
919 /* Is Root Port an agent that sends error message? */
920 result = find_device_iter(dev, e_info);
921 if (result)
922 return true;
923
924 if (pci_pcie_type(parent) == PCI_EXP_TYPE_RC_EC)
925 pcie_walk_rcec(parent, find_device_iter, e_info);
926 else
927 pci_walk_bus(parent->subordinate, find_device_iter, e_info);
928
929 if (!e_info->error_dev_num) {
930 u8 bus = e_info->id >> 8;
931 u8 devfn = e_info->id & 0xff;
932
933 pci_info(parent, "found no error details for %04x:%02x:%02x.%d\n",
934 pci_domain_nr(parent->bus), bus, PCI_SLOT(devfn),
935 PCI_FUNC(devfn));
936 return false;
937 }
938 return true;
939 }
940
941 #ifdef CONFIG_PCIEAER_CXL
942
943 /**
944 * pci_aer_unmask_internal_errors - unmask internal errors
945 * @dev: pointer to the pci_dev data structure
946 *
947 * Unmask internal errors in the Uncorrectable and Correctable Error
948 * Mask registers.
949 *
950 * Note: AER must be enabled and supported by the device which must be
951 * checked in advance, e.g. with pcie_aer_is_native().
952 */
pci_aer_unmask_internal_errors(struct pci_dev * dev)953 static void pci_aer_unmask_internal_errors(struct pci_dev *dev)
954 {
955 int aer = dev->aer_cap;
956 u32 mask;
957
958 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask);
959 mask &= ~PCI_ERR_UNC_INTN;
960 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, mask);
961
962 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask);
963 mask &= ~PCI_ERR_COR_INTERNAL;
964 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, mask);
965 }
966
is_cxl_mem_dev(struct pci_dev * dev)967 static bool is_cxl_mem_dev(struct pci_dev *dev)
968 {
969 /*
970 * The capability, status, and control fields in Device 0,
971 * Function 0 DVSEC control the CXL functionality of the
972 * entire device (CXL 3.0, 8.1.3).
973 */
974 if (dev->devfn != PCI_DEVFN(0, 0))
975 return false;
976
977 /*
978 * CXL Memory Devices must have the 502h class code set (CXL
979 * 3.0, 8.1.12.1).
980 */
981 if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL)
982 return false;
983
984 return true;
985 }
986
cxl_error_is_native(struct pci_dev * dev)987 static bool cxl_error_is_native(struct pci_dev *dev)
988 {
989 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
990
991 return (pcie_ports_native || host->native_aer);
992 }
993
is_internal_error(struct aer_err_info * info)994 static bool is_internal_error(struct aer_err_info *info)
995 {
996 if (info->severity == AER_CORRECTABLE)
997 return info->status & PCI_ERR_COR_INTERNAL;
998
999 return info->status & PCI_ERR_UNC_INTN;
1000 }
1001
cxl_rch_handle_error_iter(struct pci_dev * dev,void * data)1002 static int cxl_rch_handle_error_iter(struct pci_dev *dev, void *data)
1003 {
1004 struct aer_err_info *info = (struct aer_err_info *)data;
1005 const struct pci_error_handlers *err_handler;
1006
1007 if (!is_cxl_mem_dev(dev) || !cxl_error_is_native(dev))
1008 return 0;
1009
1010 /* Protect dev->driver */
1011 device_lock(&dev->dev);
1012
1013 err_handler = dev->driver ? dev->driver->err_handler : NULL;
1014 if (!err_handler)
1015 goto out;
1016
1017 if (info->severity == AER_CORRECTABLE) {
1018 if (err_handler->cor_error_detected)
1019 err_handler->cor_error_detected(dev);
1020 } else if (err_handler->error_detected) {
1021 if (info->severity == AER_NONFATAL)
1022 err_handler->error_detected(dev, pci_channel_io_normal);
1023 else if (info->severity == AER_FATAL)
1024 err_handler->error_detected(dev, pci_channel_io_frozen);
1025 }
1026 out:
1027 device_unlock(&dev->dev);
1028 return 0;
1029 }
1030
cxl_rch_handle_error(struct pci_dev * dev,struct aer_err_info * info)1031 static void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info)
1032 {
1033 /*
1034 * Internal errors of an RCEC indicate an AER error in an
1035 * RCH's downstream port. Check and handle them in the CXL.mem
1036 * device driver.
1037 */
1038 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC &&
1039 is_internal_error(info))
1040 pcie_walk_rcec(dev, cxl_rch_handle_error_iter, info);
1041 }
1042
handles_cxl_error_iter(struct pci_dev * dev,void * data)1043 static int handles_cxl_error_iter(struct pci_dev *dev, void *data)
1044 {
1045 bool *handles_cxl = data;
1046
1047 if (!*handles_cxl)
1048 *handles_cxl = is_cxl_mem_dev(dev) && cxl_error_is_native(dev);
1049
1050 /* Non-zero terminates iteration */
1051 return *handles_cxl;
1052 }
1053
handles_cxl_errors(struct pci_dev * rcec)1054 static bool handles_cxl_errors(struct pci_dev *rcec)
1055 {
1056 bool handles_cxl = false;
1057
1058 if (pci_pcie_type(rcec) == PCI_EXP_TYPE_RC_EC &&
1059 pcie_aer_is_native(rcec))
1060 pcie_walk_rcec(rcec, handles_cxl_error_iter, &handles_cxl);
1061
1062 return handles_cxl;
1063 }
1064
cxl_rch_enable_rcec(struct pci_dev * rcec)1065 static void cxl_rch_enable_rcec(struct pci_dev *rcec)
1066 {
1067 if (!handles_cxl_errors(rcec))
1068 return;
1069
1070 pci_aer_unmask_internal_errors(rcec);
1071 pci_info(rcec, "CXL: Internal errors unmasked");
1072 }
1073
1074 #else
cxl_rch_enable_rcec(struct pci_dev * dev)1075 static inline void cxl_rch_enable_rcec(struct pci_dev *dev) { }
cxl_rch_handle_error(struct pci_dev * dev,struct aer_err_info * info)1076 static inline void cxl_rch_handle_error(struct pci_dev *dev,
1077 struct aer_err_info *info) { }
1078 #endif
1079
1080 /**
1081 * pci_aer_handle_error - handle logging error into an event log
1082 * @dev: pointer to pci_dev data structure of error source device
1083 * @info: comprehensive error information
1084 *
1085 * Invoked when an error being detected by Root Port.
1086 */
pci_aer_handle_error(struct pci_dev * dev,struct aer_err_info * info)1087 static void pci_aer_handle_error(struct pci_dev *dev, struct aer_err_info *info)
1088 {
1089 int aer = dev->aer_cap;
1090
1091 if (info->severity == AER_CORRECTABLE) {
1092 /*
1093 * Correctable error does not need software intervention.
1094 * No need to go through error recovery process.
1095 */
1096 if (aer)
1097 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS,
1098 info->status);
1099 if (pcie_aer_is_native(dev)) {
1100 struct pci_driver *pdrv = dev->driver;
1101
1102 if (pdrv && pdrv->err_handler &&
1103 pdrv->err_handler->cor_error_detected)
1104 pdrv->err_handler->cor_error_detected(dev);
1105 pcie_clear_device_status(dev);
1106 }
1107 } else if (info->severity == AER_NONFATAL)
1108 pcie_do_recovery(dev, pci_channel_io_normal, aer_root_reset);
1109 else if (info->severity == AER_FATAL)
1110 pcie_do_recovery(dev, pci_channel_io_frozen, aer_root_reset);
1111 }
1112
handle_error_source(struct pci_dev * dev,struct aer_err_info * info)1113 static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info)
1114 {
1115 cxl_rch_handle_error(dev, info);
1116 pci_aer_handle_error(dev, info);
1117 pci_dev_put(dev);
1118 }
1119
1120 #ifdef CONFIG_ACPI_APEI_PCIEAER
1121
1122 #define AER_RECOVER_RING_SIZE 16
1123
1124 struct aer_recover_entry {
1125 u8 bus;
1126 u8 devfn;
1127 u16 domain;
1128 int severity;
1129 struct aer_capability_regs *regs;
1130 };
1131
1132 static DEFINE_KFIFO(aer_recover_ring, struct aer_recover_entry,
1133 AER_RECOVER_RING_SIZE);
1134
aer_recover_work_func(struct work_struct * work)1135 static void aer_recover_work_func(struct work_struct *work)
1136 {
1137 struct aer_recover_entry entry;
1138 struct pci_dev *pdev;
1139
1140 while (kfifo_get(&aer_recover_ring, &entry)) {
1141 pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus,
1142 entry.devfn);
1143 if (!pdev) {
1144 pr_err("no pci_dev for %04x:%02x:%02x.%x\n",
1145 entry.domain, entry.bus,
1146 PCI_SLOT(entry.devfn), PCI_FUNC(entry.devfn));
1147 continue;
1148 }
1149 pci_print_aer(pdev, entry.severity, entry.regs);
1150
1151 /*
1152 * Memory for aer_capability_regs(entry.regs) is being
1153 * allocated from the ghes_estatus_pool to protect it from
1154 * overwriting when multiple sections are present in the
1155 * error status. Thus free the same after processing the
1156 * data.
1157 */
1158 ghes_estatus_pool_region_free((unsigned long)entry.regs,
1159 sizeof(struct aer_capability_regs));
1160
1161 if (entry.severity == AER_NONFATAL)
1162 pcie_do_recovery(pdev, pci_channel_io_normal,
1163 aer_root_reset);
1164 else if (entry.severity == AER_FATAL)
1165 pcie_do_recovery(pdev, pci_channel_io_frozen,
1166 aer_root_reset);
1167 pci_dev_put(pdev);
1168 }
1169 }
1170
1171 /*
1172 * Mutual exclusion for writers of aer_recover_ring, reader side don't
1173 * need lock, because there is only one reader and lock is not needed
1174 * between reader and writer.
1175 */
1176 static DEFINE_SPINLOCK(aer_recover_ring_lock);
1177 static DECLARE_WORK(aer_recover_work, aer_recover_work_func);
1178
aer_recover_queue(int domain,unsigned int bus,unsigned int devfn,int severity,struct aer_capability_regs * aer_regs)1179 void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn,
1180 int severity, struct aer_capability_regs *aer_regs)
1181 {
1182 struct aer_recover_entry entry = {
1183 .bus = bus,
1184 .devfn = devfn,
1185 .domain = domain,
1186 .severity = severity,
1187 .regs = aer_regs,
1188 };
1189
1190 if (kfifo_in_spinlocked(&aer_recover_ring, &entry, 1,
1191 &aer_recover_ring_lock))
1192 schedule_work(&aer_recover_work);
1193 else
1194 pr_err("buffer overflow in recovery for %04x:%02x:%02x.%x\n",
1195 domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1196 }
1197 EXPORT_SYMBOL_GPL(aer_recover_queue);
1198 #endif
1199
1200 /**
1201 * aer_get_device_error_info - read error status from dev and store it to info
1202 * @dev: pointer to the device expected to have an error record
1203 * @info: pointer to structure to store the error record
1204 *
1205 * Return: 1 on success, 0 on error.
1206 *
1207 * Note that @info is reused among all error devices. Clear fields properly.
1208 */
aer_get_device_error_info(struct pci_dev * dev,struct aer_err_info * info)1209 int aer_get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
1210 {
1211 int type = pci_pcie_type(dev);
1212 int aer = dev->aer_cap;
1213 u32 aercc;
1214
1215 /* Must reset in this function */
1216 info->status = 0;
1217 info->tlp_header_valid = 0;
1218
1219 /* The device might not support AER */
1220 if (!aer)
1221 return 0;
1222
1223 if (info->severity == AER_CORRECTABLE) {
1224 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS,
1225 &info->status);
1226 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK,
1227 &info->mask);
1228 if (!(info->status & ~info->mask))
1229 return 0;
1230 } else if (type == PCI_EXP_TYPE_ROOT_PORT ||
1231 type == PCI_EXP_TYPE_RC_EC ||
1232 type == PCI_EXP_TYPE_DOWNSTREAM ||
1233 info->severity == AER_NONFATAL) {
1234
1235 /* Link is still healthy for IO reads */
1236 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS,
1237 &info->status);
1238 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK,
1239 &info->mask);
1240 if (!(info->status & ~info->mask))
1241 return 0;
1242
1243 /* Get First Error Pointer */
1244 pci_read_config_dword(dev, aer + PCI_ERR_CAP, &aercc);
1245 info->first_error = PCI_ERR_CAP_FEP(aercc);
1246
1247 if (info->status & AER_LOG_TLP_MASKS) {
1248 info->tlp_header_valid = 1;
1249 pcie_read_tlp_log(dev, aer + PCI_ERR_HEADER_LOG,
1250 aer + PCI_ERR_PREFIX_LOG,
1251 aer_tlp_log_len(dev, aercc),
1252 aercc & PCI_ERR_CAP_TLP_LOG_FLIT,
1253 &info->tlp);
1254 }
1255 }
1256
1257 return 1;
1258 }
1259
aer_process_err_devices(struct aer_err_info * e_info)1260 static inline void aer_process_err_devices(struct aer_err_info *e_info)
1261 {
1262 int i;
1263
1264 /* Report all before handling them, to not lose records by reset etc. */
1265 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {
1266 if (aer_get_device_error_info(e_info->dev[i], e_info))
1267 aer_print_error(e_info->dev[i], e_info);
1268 }
1269 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {
1270 if (aer_get_device_error_info(e_info->dev[i], e_info))
1271 handle_error_source(e_info->dev[i], e_info);
1272 }
1273 }
1274
1275 /**
1276 * aer_isr_one_error - consume an error detected by Root Port
1277 * @rpc: pointer to the Root Port which holds an error
1278 * @e_src: pointer to an error source
1279 */
aer_isr_one_error(struct aer_rpc * rpc,struct aer_err_source * e_src)1280 static void aer_isr_one_error(struct aer_rpc *rpc,
1281 struct aer_err_source *e_src)
1282 {
1283 struct pci_dev *pdev = rpc->rpd;
1284 struct aer_err_info e_info;
1285
1286 pci_rootport_aer_stats_incr(pdev, e_src);
1287
1288 /*
1289 * There is a possibility that both correctable error and
1290 * uncorrectable error being logged. Report correctable error first.
1291 */
1292 if (e_src->status & PCI_ERR_ROOT_COR_RCV) {
1293 e_info.id = ERR_COR_ID(e_src->id);
1294 e_info.severity = AER_CORRECTABLE;
1295
1296 if (e_src->status & PCI_ERR_ROOT_MULTI_COR_RCV)
1297 e_info.multi_error_valid = 1;
1298 else
1299 e_info.multi_error_valid = 0;
1300 aer_print_port_info(pdev, &e_info);
1301
1302 if (find_source_device(pdev, &e_info))
1303 aer_process_err_devices(&e_info);
1304 }
1305
1306 if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) {
1307 e_info.id = ERR_UNCOR_ID(e_src->id);
1308
1309 if (e_src->status & PCI_ERR_ROOT_FATAL_RCV)
1310 e_info.severity = AER_FATAL;
1311 else
1312 e_info.severity = AER_NONFATAL;
1313
1314 if (e_src->status & PCI_ERR_ROOT_MULTI_UNCOR_RCV)
1315 e_info.multi_error_valid = 1;
1316 else
1317 e_info.multi_error_valid = 0;
1318
1319 aer_print_port_info(pdev, &e_info);
1320
1321 if (find_source_device(pdev, &e_info))
1322 aer_process_err_devices(&e_info);
1323 }
1324 }
1325
1326 /**
1327 * aer_isr - consume errors detected by Root Port
1328 * @irq: IRQ assigned to Root Port
1329 * @context: pointer to Root Port data structure
1330 *
1331 * Invoked, as DPC, when Root Port records new detected error
1332 */
aer_isr(int irq,void * context)1333 static irqreturn_t aer_isr(int irq, void *context)
1334 {
1335 struct pcie_device *dev = (struct pcie_device *)context;
1336 struct aer_rpc *rpc = get_service_data(dev);
1337 struct aer_err_source e_src;
1338
1339 if (kfifo_is_empty(&rpc->aer_fifo))
1340 return IRQ_NONE;
1341
1342 while (kfifo_get(&rpc->aer_fifo, &e_src))
1343 aer_isr_one_error(rpc, &e_src);
1344 return IRQ_HANDLED;
1345 }
1346
1347 /**
1348 * aer_irq - Root Port's ISR
1349 * @irq: IRQ assigned to Root Port
1350 * @context: pointer to Root Port data structure
1351 *
1352 * Invoked when Root Port detects AER messages.
1353 */
aer_irq(int irq,void * context)1354 static irqreturn_t aer_irq(int irq, void *context)
1355 {
1356 struct pcie_device *pdev = (struct pcie_device *)context;
1357 struct aer_rpc *rpc = get_service_data(pdev);
1358 struct pci_dev *rp = rpc->rpd;
1359 int aer = rp->aer_cap;
1360 struct aer_err_source e_src = {};
1361
1362 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, &e_src.status);
1363 if (!(e_src.status & AER_ERR_STATUS_MASK))
1364 return IRQ_NONE;
1365
1366 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_ERR_SRC, &e_src.id);
1367 pci_write_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, e_src.status);
1368
1369 if (!kfifo_put(&rpc->aer_fifo, e_src))
1370 return IRQ_HANDLED;
1371
1372 return IRQ_WAKE_THREAD;
1373 }
1374
aer_enable_irq(struct pci_dev * pdev)1375 static void aer_enable_irq(struct pci_dev *pdev)
1376 {
1377 int aer = pdev->aer_cap;
1378 u32 reg32;
1379
1380 /* Enable Root Port's interrupt in response to error messages */
1381 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32);
1382 reg32 |= ROOT_PORT_INTR_ON_MESG_MASK;
1383 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32);
1384 }
1385
aer_disable_irq(struct pci_dev * pdev)1386 static void aer_disable_irq(struct pci_dev *pdev)
1387 {
1388 int aer = pdev->aer_cap;
1389 u32 reg32;
1390
1391 /* Disable Root Port's interrupt in response to error messages */
1392 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32);
1393 reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK;
1394 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32);
1395 }
1396
1397 /**
1398 * aer_enable_rootport - enable Root Port's interrupts when receiving messages
1399 * @rpc: pointer to a Root Port data structure
1400 *
1401 * Invoked when PCIe bus loads AER service driver.
1402 */
aer_enable_rootport(struct aer_rpc * rpc)1403 static void aer_enable_rootport(struct aer_rpc *rpc)
1404 {
1405 struct pci_dev *pdev = rpc->rpd;
1406 int aer = pdev->aer_cap;
1407 u16 reg16;
1408 u32 reg32;
1409
1410 /* Clear PCIe Capability's Device Status */
1411 pcie_capability_read_word(pdev, PCI_EXP_DEVSTA, ®16);
1412 pcie_capability_write_word(pdev, PCI_EXP_DEVSTA, reg16);
1413
1414 /* Disable system error generation in response to error messages */
1415 pcie_capability_clear_word(pdev, PCI_EXP_RTCTL,
1416 SYSTEM_ERROR_INTR_ON_MESG_MASK);
1417
1418 /* Clear error status */
1419 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32);
1420 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32);
1421 pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, ®32);
1422 pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32);
1423 pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, ®32);
1424 pci_write_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, reg32);
1425
1426 aer_enable_irq(pdev);
1427 }
1428
1429 /**
1430 * aer_disable_rootport - disable Root Port's interrupts when receiving messages
1431 * @rpc: pointer to a Root Port data structure
1432 *
1433 * Invoked when PCIe bus unloads AER service driver.
1434 */
aer_disable_rootport(struct aer_rpc * rpc)1435 static void aer_disable_rootport(struct aer_rpc *rpc)
1436 {
1437 struct pci_dev *pdev = rpc->rpd;
1438 int aer = pdev->aer_cap;
1439 u32 reg32;
1440
1441 aer_disable_irq(pdev);
1442
1443 /* Clear Root's error status reg */
1444 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32);
1445 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32);
1446 }
1447
1448 /**
1449 * aer_remove - clean up resources
1450 * @dev: pointer to the pcie_dev data structure
1451 *
1452 * Invoked when PCI Express bus unloads or AER probe fails.
1453 */
aer_remove(struct pcie_device * dev)1454 static void aer_remove(struct pcie_device *dev)
1455 {
1456 struct aer_rpc *rpc = get_service_data(dev);
1457
1458 aer_disable_rootport(rpc);
1459 }
1460
1461 /**
1462 * aer_probe - initialize resources
1463 * @dev: pointer to the pcie_dev data structure
1464 *
1465 * Invoked when PCI Express bus loads AER service driver.
1466 */
aer_probe(struct pcie_device * dev)1467 static int aer_probe(struct pcie_device *dev)
1468 {
1469 int status;
1470 struct aer_rpc *rpc;
1471 struct device *device = &dev->device;
1472 struct pci_dev *port = dev->port;
1473
1474 BUILD_BUG_ON(ARRAY_SIZE(aer_correctable_error_string) <
1475 AER_MAX_TYPEOF_COR_ERRS);
1476 BUILD_BUG_ON(ARRAY_SIZE(aer_uncorrectable_error_string) <
1477 AER_MAX_TYPEOF_UNCOR_ERRS);
1478
1479 /* Limit to Root Ports or Root Complex Event Collectors */
1480 if ((pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC) &&
1481 (pci_pcie_type(port) != PCI_EXP_TYPE_ROOT_PORT))
1482 return -ENODEV;
1483
1484 rpc = devm_kzalloc(device, sizeof(struct aer_rpc), GFP_KERNEL);
1485 if (!rpc)
1486 return -ENOMEM;
1487
1488 rpc->rpd = port;
1489 INIT_KFIFO(rpc->aer_fifo);
1490 set_service_data(dev, rpc);
1491
1492 status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr,
1493 IRQF_SHARED, "aerdrv", dev);
1494 if (status) {
1495 pci_err(port, "request AER IRQ %d failed\n", dev->irq);
1496 return status;
1497 }
1498
1499 cxl_rch_enable_rcec(port);
1500 aer_enable_rootport(rpc);
1501 pci_info(port, "enabled with IRQ %d\n", dev->irq);
1502 return 0;
1503 }
1504
aer_suspend(struct pcie_device * dev)1505 static int aer_suspend(struct pcie_device *dev)
1506 {
1507 struct aer_rpc *rpc = get_service_data(dev);
1508
1509 aer_disable_rootport(rpc);
1510 return 0;
1511 }
1512
aer_resume(struct pcie_device * dev)1513 static int aer_resume(struct pcie_device *dev)
1514 {
1515 struct aer_rpc *rpc = get_service_data(dev);
1516
1517 aer_enable_rootport(rpc);
1518 return 0;
1519 }
1520
1521 /**
1522 * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP
1523 * @dev: pointer to Root Port, RCEC, or RCiEP
1524 *
1525 * Invoked by Port Bus driver when performing reset.
1526 */
aer_root_reset(struct pci_dev * dev)1527 static pci_ers_result_t aer_root_reset(struct pci_dev *dev)
1528 {
1529 int type = pci_pcie_type(dev);
1530 struct pci_dev *root;
1531 int aer;
1532 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
1533 u32 reg32;
1534 int rc;
1535
1536 /*
1537 * Only Root Ports and RCECs have AER Root Command and Root Status
1538 * registers. If "dev" is an RCiEP, the relevant registers are in
1539 * the RCEC.
1540 */
1541 if (type == PCI_EXP_TYPE_RC_END)
1542 root = dev->rcec;
1543 else
1544 root = pcie_find_root_port(dev);
1545
1546 /*
1547 * If the platform retained control of AER, an RCiEP may not have
1548 * an RCEC visible to us, so dev->rcec ("root") may be NULL. In
1549 * that case, firmware is responsible for these registers.
1550 */
1551 aer = root ? root->aer_cap : 0;
1552
1553 if ((host->native_aer || pcie_ports_native) && aer)
1554 aer_disable_irq(root);
1555
1556 if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) {
1557 rc = pcie_reset_flr(dev, PCI_RESET_DO_RESET);
1558 if (!rc)
1559 pci_info(dev, "has been reset\n");
1560 else
1561 pci_info(dev, "not reset (no FLR support: %d)\n", rc);
1562 } else {
1563 rc = pci_bus_error_reset(dev);
1564 pci_info(dev, "%s Port link has been reset (%d)\n",
1565 pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc);
1566 }
1567
1568 if ((host->native_aer || pcie_ports_native) && aer) {
1569 /* Clear Root Error Status */
1570 pci_read_config_dword(root, aer + PCI_ERR_ROOT_STATUS, ®32);
1571 pci_write_config_dword(root, aer + PCI_ERR_ROOT_STATUS, reg32);
1572
1573 aer_enable_irq(root);
1574 }
1575
1576 return rc ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
1577 }
1578
1579 static struct pcie_port_service_driver aerdriver = {
1580 .name = "aer",
1581 .port_type = PCIE_ANY_PORT,
1582 .service = PCIE_PORT_SERVICE_AER,
1583
1584 .probe = aer_probe,
1585 .suspend = aer_suspend,
1586 .resume = aer_resume,
1587 .remove = aer_remove,
1588 };
1589
1590 /**
1591 * pcie_aer_init - register AER service driver
1592 *
1593 * Invoked when AER service driver is loaded.
1594 */
pcie_aer_init(void)1595 int __init pcie_aer_init(void)
1596 {
1597 if (!pci_aer_available())
1598 return -ENXIO;
1599 return pcie_port_service_register(&aerdriver);
1600 }
1601