From 459f863b07761995006d894f5035e37d1bbd4e64 Mon Sep 17 00:00:00 2001 From: James Sewart Date: Tue, 18 Nov 2025 16:18:00 +0000 Subject: [PATCH] PCIe AER printk ratelimiting backport --- ...OR-UNCOR-error-handling-out-from-aer.patch | 104 +++++++ ...-Fix-NULL-pointer-access-by-aer_info.patch | 36 +++ .../0001-PCI-AER-Simplify-pci_print_aer.patch | 69 +++++ ...LL-pointer-dereference-in-aer_rateli.patch | 61 ++++ ...pdate-statistics-before-ratelimiting.patch | 68 +++++ ...race-error-event-before-ratelimiting.patch | 69 +++++ ...04-PCI-AER-Simplify-add_error_device.patch | 49 ++++ ...Check-log-level-once-and-remember-it.patch | 128 +++++++++ ...aer_get_device_error_info-aer_print_.patch | 150 ++++++++++ ...t-correctable-and-non-fatal-error-lo.patch | 221 ++++++++++++++ ...-sysfs-attributes-for-log-ratelimits.patch | 272 ++++++++++++++++++ ...-aer-ratelimiting-sysfs-output-units.patch | 33 +++ patches-sonic/series | 14 + 13 files changed, 1274 insertions(+) create mode 100644 patches-sonic/0001-PCI-AER-Factor-COR-UNCOR-error-handling-out-from-aer.patch create mode 100644 patches-sonic/0001-PCI-AER-Fix-NULL-pointer-access-by-aer_info.patch create mode 100644 patches-sonic/0001-PCI-AER-Simplify-pci_print_aer.patch create mode 100644 patches-sonic/0002-PCI-AER-Avoid-NULL-pointer-dereference-in-aer_rateli.patch create mode 100644 patches-sonic/0002-PCI-AER-Update-statistics-before-ratelimiting.patch create mode 100644 patches-sonic/0003-PCI-AER-Trace-error-event-before-ratelimiting.patch create mode 100644 patches-sonic/0004-PCI-AER-Simplify-add_error_device.patch create mode 100644 patches-sonic/0005-PCI-AER-Check-log-level-once-and-remember-it.patch create mode 100644 patches-sonic/0006-PCI-AER-Convert-aer_get_device_error_info-aer_print_.patch create mode 100644 patches-sonic/0007-PCI-AER-Ratelimit-correctable-and-non-fatal-error-lo.patch create mode 100644 patches-sonic/0008-PCI-AER-Add-sysfs-attributes-for-log-ratelimits.patch create mode 100644 patches-sonic/fix-aer-ratelimiting-sysfs-output-units.patch diff --git a/patches-sonic/0001-PCI-AER-Factor-COR-UNCOR-error-handling-out-from-aer.patch b/patches-sonic/0001-PCI-AER-Factor-COR-UNCOR-error-handling-out-from-aer.patch new file mode 100644 index 000000000..a5ec4c52c --- /dev/null +++ b/patches-sonic/0001-PCI-AER-Factor-COR-UNCOR-error-handling-out-from-aer.patch @@ -0,0 +1,104 @@ +From 8aec28caf011788522c1fd2dea92959477016053 Mon Sep 17 00:00:00 2001 +From: Bjorn Helgaas +Date: Thu, 22 May 2025 18:21:09 -0500 +Subject: [PATCH] PCI/AER: Factor COR/UNCOR error handling out from + aer_isr_one_error() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +aer_isr_one_error() duplicates the Error Source ID logging and AER error +processing for Correctable Errors and Uncorrectable Errors. Factor out the +duplicated code to aer_isr_one_error_type(). + +aer_isr_one_error() doesn't need the struct aer_rpc pointer, so pass it the +Root Port or RCEC pci_dev pointer instead. + +Signed-off-by: Bjorn Helgaas +Reviewed-by: Ilpo Järvinen +Reviewed-by: Jonathan Cameron +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-4-helgaas@kernel.org +(cherry picked from commit 6fc4dae74afcf29ef82afbaaa9b082893871eda4) +--- + drivers/pci/pcie/aer.c | 36 +++++++++++++++++++++++------------- + 1 file changed, 23 insertions(+), 13 deletions(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -1287,17 +1287,32 @@ static inline void aer_process_err_devices(struct aer_err_info *e_info) + } + + /** +- * aer_isr_one_error - consume an error detected by root port +- * @rpc: pointer to the root port which holds an error ++ * aer_isr_one_error_type - consume a Correctable or Uncorrectable Error ++ * detected by Root Port or RCEC ++ * @root: pointer to Root Port or RCEC that signaled AER interrupt ++ * @info: pointer to AER error info ++ */ ++static void aer_isr_one_error_type(struct pci_dev *root, ++ struct aer_err_info *info) ++{ ++ aer_print_port_info(root, info); ++ ++ if (find_source_device(root, info)) ++ aer_process_err_devices(info); ++} ++ ++/** ++ * aer_isr_one_error - consume error(s) signaled by an AER interrupt from ++ * Root Port or RCEC ++ * @root: pointer to Root Port or RCEC that signaled AER interrupt + * @e_src: pointer to an error source + */ +-static void aer_isr_one_error(struct aer_rpc *rpc, ++static void aer_isr_one_error(struct pci_dev *root, + struct aer_err_source *e_src) + { +- struct pci_dev *pdev = rpc->rpd; + struct aer_err_info e_info; + +- pci_rootport_aer_stats_incr(pdev, e_src); ++ pci_rootport_aer_stats_incr(root, e_src); + + /* + * There is a possibility that both correctable error and +@@ -1312,10 +1327,8 @@ static void aer_isr_one_error(struct aer_rpc *rpc, + e_info.multi_error_valid = 1; + else + e_info.multi_error_valid = 0; +- aer_print_port_info(pdev, &e_info); + +- if (find_source_device(pdev, &e_info)) +- aer_process_err_devices(&e_info); ++ aer_isr_one_error_type(root, &e_info); + } + + if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { +@@ -1332,10 +1345,7 @@ static void aer_isr_one_error(struct aer_rpc *rpc, + else + e_info.multi_error_valid = 0; + +- aer_print_port_info(pdev, &e_info); +- +- if (find_source_device(pdev, &e_info)) +- aer_process_err_devices(&e_info); ++ aer_isr_one_error_type(root, &e_info); + } + } + +@@ -1356,7 +1366,7 @@ static irqreturn_t aer_isr(int irq, void *context) + return IRQ_NONE; + + while (kfifo_get(&rpc->aer_fifo, &e_src)) +- aer_isr_one_error(rpc, &e_src); ++ aer_isr_one_error(rpc->rpd, &e_src); + return IRQ_HANDLED; + } + +-- +2.47.0 + diff --git a/patches-sonic/0001-PCI-AER-Fix-NULL-pointer-access-by-aer_info.patch b/patches-sonic/0001-PCI-AER-Fix-NULL-pointer-access-by-aer_info.patch new file mode 100644 index 000000000..5089ff321 --- /dev/null +++ b/patches-sonic/0001-PCI-AER-Fix-NULL-pointer-access-by-aer_info.patch @@ -0,0 +1,36 @@ +From cf5770619326108794a72ca7b3500ad9e3aefa90 Mon Sep 17 00:00:00 2001 +From: Vernon Yang +Date: Fri, 5 Sep 2025 02:25:27 +0800 +Subject: [PATCH 1/2] PCI/AER: Fix NULL pointer access by aer_info + +The kzalloc(GFP_KERNEL) may return NULL, so all accesses to aer_info->xxx +will result in kernel panic. Fix it. + +Signed-off-by: Vernon Yang +Signed-off-by: Bjorn Helgaas +Link: https://patch.msgid.link/20250904182527.67371-1-vernon2gm@gmail.com +(cherry picked from commit 0a27bdb14b028fed30a10cec2f945c38cb5ca4fa) +--- + drivers/pci/pcie/aer.c | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -385,7 +385,11 @@ void pci_aer_init(struct pci_dev *dev) + if (!dev->aer_cap) + return; + +- dev->aer_stats = kzalloc(sizeof(struct aer_stats), GFP_KERNEL); ++ dev->aer_stats = kzalloc(sizeof(*dev->aer_stats), GFP_KERNEL); ++ if (!dev->aer_stats) { ++ dev->aer_cap = 0; ++ return; ++ } + + ratelimit_state_init(&dev->aer_stats->correctable_ratelimit, + DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); +-- +2.39.5 (Apple Git-154) + diff --git a/patches-sonic/0001-PCI-AER-Simplify-pci_print_aer.patch b/patches-sonic/0001-PCI-AER-Simplify-pci_print_aer.patch new file mode 100644 index 000000000..4e028f95b --- /dev/null +++ b/patches-sonic/0001-PCI-AER-Simplify-pci_print_aer.patch @@ -0,0 +1,69 @@ +From fed119ab131c202e7677ce0228d17f5cb74baa29 Mon Sep 17 00:00:00 2001 +From: Bjorn Helgaas +Date: Thu, 22 May 2025 18:21:15 -0500 +Subject: [PATCH 1/8] PCI/AER: Simplify pci_print_aer() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Simplify pci_print_aer() by initializing the struct aer_err_info "info" +with a designated initializer list (it was previously initialized with +memset()) and using pci_name(). + +Signed-off-by: Bjorn Helgaas +Tested-by: Krzysztof Wilczyński +Reviewed-by: Ilpo Järvinen +Reviewed-by: Jonathan Cameron +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-10-helgaas@kernel.org +(cherry picked from commit ad9839137cf9fb0f0c2d531bd04bc4382e6f2de9) +--- + drivers/pci/pcie/aer.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -774,7 +774,10 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + { + int layer, agent, tlp_header_valid = 0; + u32 status, mask; +- struct aer_err_info info; ++ struct aer_err_info info = { ++ .severity = aer_severity, ++ .first_error = PCI_ERR_CAP_FEP(aer->cap_control), ++ }; + + if (aer_severity == AER_CORRECTABLE) { + status = aer->cor_status; +@@ -785,14 +788,11 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + tlp_header_valid = status & AER_LOG_TLP_MASKS; + } + +- layer = AER_GET_LAYER_ERROR(aer_severity, status); +- agent = AER_GET_AGENT(aer_severity, status); +- +- memset(&info, 0, sizeof(info)); +- info.severity = aer_severity; + info.status = status; + info.mask = mask; +- info.first_error = PCI_ERR_CAP_FEP(aer->cap_control); ++ ++ layer = AER_GET_LAYER_ERROR(aer_severity, status); ++ agent = AER_GET_AGENT(aer_severity, status); + + pci_err(dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n", status, mask); + __aer_print_error(dev, &info); +@@ -806,7 +806,7 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + if (tlp_header_valid) + __print_tlp_header(dev, &aer->header_log); + +- trace_aer_event(dev_name(&dev->dev), (status & ~mask), ++ trace_aer_event(pci_name(dev), (status & ~mask), + aer_severity, tlp_header_valid, &aer->header_log); + } + #endif +-- +2.47.0 + diff --git a/patches-sonic/0002-PCI-AER-Avoid-NULL-pointer-dereference-in-aer_rateli.patch b/patches-sonic/0002-PCI-AER-Avoid-NULL-pointer-dereference-in-aer_rateli.patch new file mode 100644 index 000000000..b196154f0 --- /dev/null +++ b/patches-sonic/0002-PCI-AER-Avoid-NULL-pointer-dereference-in-aer_rateli.patch @@ -0,0 +1,61 @@ +From 23d8218139853dda49859e3041f17111cdc47400 Mon Sep 17 00:00:00 2001 +From: Breno Leitao +Date: Mon, 29 Sep 2025 02:15:47 -0700 +Subject: [PATCH 2/2] PCI/AER: Avoid NULL pointer dereference in + aer_ratelimit() + +When platform firmware supplies error information to the OS, e.g., via the +ACPI APEI GHES mechanism, it may identify an error source device that +doesn't advertise an AER Capability and therefore dev->aer_info, which +contains AER stats and ratelimiting data, is NULL. + +pci_dev_aer_stats_incr() already checks dev->aer_info for NULL, but +aer_ratelimit() did not, leading to NULL pointer dereferences like this one +from the URL below: + + {1}[Hardware Error]: Hardware error from APEI Generic Hardware Error Source: 0 + {1}[Hardware Error]: event severity: corrected + {1}[Hardware Error]: device_id: 0000:00:00.0 + {1}[Hardware Error]: vendor_id: 0x8086, device_id: 0x2020 + {1}[Hardware Error]: aer_cor_status: 0x00001000, aer_cor_mask: 0x00002000 + BUG: kernel NULL pointer dereference, address: 0000000000000264 + RIP: 0010:___ratelimit+0xc/0x1b0 + pci_print_aer+0x141/0x360 + aer_recover_work_func+0xb5/0x130 + +[8086:2020] is an Intel "Sky Lake-E DMI3 Registers" device that claims to +be a Root Port but does not advertise an AER Capability. + +Add a NULL check in aer_ratelimit() to avoid the NULL pointer dereference. +Note that this also prevents ratelimiting these events from GHES. + +Fixes: a57f2bfb4a5863 ("PCI/AER: Ratelimit correctable and non-fatal error logging") +Link: https://lore.kernel.org/r/buduna6darbvwfg3aogl5kimyxkggu3n4romnmq6sozut6axeu@clnx7sfsy457/ +Signed-off-by: Breno Leitao +[bhelgaas: add crash details to commit log] +Signed-off-by: Bjorn Helgaas +Reviewed-by: Kuppuswamy Sathyanarayanan +Cc: stable@vger.kernel.org +Link: https://patch.msgid.link/20250929-aer_crash_2-v1-1-68ec4f81c356@debian.org +(cherry picked from commit deb2f228388ff3a9d0623e3b59a053e9235c341d) +--- + drivers/pci/pcie/aer.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -800,6 +800,9 @@ static void __print_tlp_header(struct pci_dev *dev, + + static int aer_ratelimit(struct pci_dev *dev, unsigned int severity) + { ++ if (!dev->aer_stats) ++ return 1; ++ + switch (severity) { + case AER_NONFATAL: + return __ratelimit(&dev->aer_stats->nonfatal_ratelimit); +-- +2.39.5 (Apple Git-154) + diff --git a/patches-sonic/0002-PCI-AER-Update-statistics-before-ratelimiting.patch b/patches-sonic/0002-PCI-AER-Update-statistics-before-ratelimiting.patch new file mode 100644 index 000000000..60d97218f --- /dev/null +++ b/patches-sonic/0002-PCI-AER-Update-statistics-before-ratelimiting.patch @@ -0,0 +1,68 @@ +From 8bbcbe849d91b64da2850797482bf0915772898b Mon Sep 17 00:00:00 2001 +From: Bjorn Helgaas +Date: Thu, 22 May 2025 18:21:16 -0500 +Subject: [PATCH 2/8] PCI/AER: Update statistics before ratelimiting +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +There are two AER logging entry points: + + - aer_print_error() is used by DPC (dpc_process_error()) and native AER + handling (aer_process_err_devices()). + + - pci_print_aer() is used by GHES (aer_recover_work_func()) and CXL + (cxl_handle_rdport_errors()) + +Both use __aer_print_error() to print the AER error bits. Previously +__aer_print_error() also incremented the AER statistics via +pci_dev_aer_stats_incr(). + +Call pci_dev_aer_stats_incr() early in the entry points instead of in +__aer_print_error() so we update the statistics even if the actual printing +of error bits is rate limited by a future change. + +Signed-off-by: Bjorn Helgaas +Tested-by: Krzysztof Wilczyński +Reviewed-by: Ilpo Järvinen +Reviewed-by: Kuppuswamy Sathyanarayanan +Reviewed-by: Jonathan Cameron +Link: https://patch.msgid.link/20250522232339.1525671-11-helgaas@kernel.org +(cherry picked from commit 88a7765e62b9e4c79c7ca2c7b749ae04f54a5668) +--- + drivers/pci/pcie/aer.c | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -703,7 +703,6 @@ static void __aer_print_error(struct pci_dev *dev, + pci_printk(level, dev, " [%2d] %-22s%s\n", i, errmsg, + info->first_error == i ? " (First)" : ""); + } +- pci_dev_aer_stats_incr(dev, info); + } + + void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) +@@ -712,6 +711,8 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) + int id = ((dev->bus->number << 8) | dev->devfn); + const char *level; + ++ pci_dev_aer_stats_incr(dev, info); ++ + if (!info->status) { + pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n", + aer_error_severity_string[info->severity]); +@@ -791,6 +792,8 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + info.status = status; + info.mask = mask; + ++ pci_dev_aer_stats_incr(dev, &info); ++ + layer = AER_GET_LAYER_ERROR(aer_severity, status); + agent = AER_GET_AGENT(aer_severity, status); + +-- +2.47.0 + diff --git a/patches-sonic/0003-PCI-AER-Trace-error-event-before-ratelimiting.patch b/patches-sonic/0003-PCI-AER-Trace-error-event-before-ratelimiting.patch new file mode 100644 index 000000000..9b2defc83 --- /dev/null +++ b/patches-sonic/0003-PCI-AER-Trace-error-event-before-ratelimiting.patch @@ -0,0 +1,69 @@ +From 9232650afd9ebf430cdf85f75bf1bf4ecce03c0f Mon Sep 17 00:00:00 2001 +From: Bjorn Helgaas +Date: Thu, 22 May 2025 18:21:17 -0500 +Subject: [PATCH 3/8] PCI/AER: Trace error event before ratelimiting +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +As with the AER statistics, we always want to emit trace events, even if +the actual dmesg logging is rate limited. + +Call trace_aer_event() immediately after pci_dev_aer_stats_incr() so both +happen before ratelimiting. + +Signed-off-by: Bjorn Helgaas +Tested-by: Krzysztof Wilczyński +Reviewed-by: Ilpo Järvinen +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-12-helgaas@kernel.org +(cherry picked from commit 6bb4befbd65fa7f99688fb707e376637e5acfe36) +--- + drivers/pci/pcie/aer.c | 10 ++++------ + 1 file changed, 4 insertions(+), 6 deletions(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -712,6 +712,8 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) + const char *level; + + pci_dev_aer_stats_incr(dev, info); ++ trace_aer_event(pci_name(dev), (info->status & ~info->mask), ++ info->severity, info->tlp_header_valid, &info->tlp); + + if (!info->status) { + pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n", +@@ -739,9 +741,6 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) + out: + if (info->id && info->error_dev_num > 1 && info->id == id) + pci_err(dev, " Error of this Agent is reported first\n"); +- +- trace_aer_event(dev_name(&dev->dev), (info->status & ~info->mask), +- info->severity, info->tlp_header_valid, &info->tlp); + } + + static void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info) +@@ -793,6 +792,8 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + info.mask = mask; + + pci_dev_aer_stats_incr(dev, &info); ++ trace_aer_event(pci_name(dev), (status & ~mask), ++ aer_severity, tlp_header_valid, &aer->header_log); + + layer = AER_GET_LAYER_ERROR(aer_severity, status); + agent = AER_GET_AGENT(aer_severity, status); +@@ -808,9 +809,6 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + + if (tlp_header_valid) + __print_tlp_header(dev, &aer->header_log); +- +- trace_aer_event(pci_name(dev), (status & ~mask), +- aer_severity, tlp_header_valid, &aer->header_log); + } + #endif + +-- +2.47.0 + diff --git a/patches-sonic/0004-PCI-AER-Simplify-add_error_device.patch b/patches-sonic/0004-PCI-AER-Simplify-add_error_device.patch new file mode 100644 index 000000000..3cf6cd40e --- /dev/null +++ b/patches-sonic/0004-PCI-AER-Simplify-add_error_device.patch @@ -0,0 +1,49 @@ +From 68b5f15d70cabfc8c9706f313d51fb82f766e4d6 Mon Sep 17 00:00:00 2001 +From: Bjorn Helgaas +Date: Thu, 22 May 2025 18:21:23 -0500 +Subject: [PATCH 4/8] PCI/AER: Simplify add_error_device() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Return -ENOSPC error early so the usual path through add_error_device() is +the straightline code. + +Signed-off-by: Bjorn Helgaas +Reviewed-by: Ilpo Järvinen +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-18-helgaas@kernel.org +(cherry picked from commit d72bae423004aa7b4d94c34a7fd0b48b64305a08) +--- + drivers/pci/pcie/aer.c | 15 +++++++++------ + 1 file changed, 9 insertions(+), 6 deletions(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -819,12 +819,15 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + */ + static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev) + { +- if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) { +- e_info->dev[e_info->error_dev_num] = pci_dev_get(dev); +- e_info->error_dev_num++; +- return 0; +- } +- return -ENOSPC; ++ int i = e_info->error_dev_num; ++ ++ if (i >= AER_MAX_MULTI_ERR_DEVICES) ++ return -ENOSPC; ++ ++ e_info->dev[i] = pci_dev_get(dev); ++ e_info->error_dev_num++; ++ ++ return 0; + } + + /** +-- +2.47.0 + diff --git a/patches-sonic/0005-PCI-AER-Check-log-level-once-and-remember-it.patch b/patches-sonic/0005-PCI-AER-Check-log-level-once-and-remember-it.patch new file mode 100644 index 000000000..c04606d49 --- /dev/null +++ b/patches-sonic/0005-PCI-AER-Check-log-level-once-and-remember-it.patch @@ -0,0 +1,128 @@ +From 826b9ac84463a15f4cb8e9cdb82ded0695fc5b4a Mon Sep 17 00:00:00 2001 +From: Karolina Stolarek +Date: Thu, 22 May 2025 18:21:18 -0500 +Subject: [PATCH 5/8] PCI/AER: Check log level once and remember it +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +When reporting an AER error, we check its type multiple times to determine +the log level for each message. Do this check only in the top-level +functions (aer_isr_one_error(), pci_print_aer()) and save the level in +struct aer_err_info. + +[bhelgaas: save log level in struct aer_err_info instead of passing it +as a parameter] + +Signed-off-by: Karolina Stolarek +Signed-off-by: Bjorn Helgaas +Tested-by: Krzysztof Wilczyński +Reviewed-by: Ilpo Järvinen +Reviewed-by: Kuppuswamy Sathyanarayanan +Reviewed-by: Jonathan Cameron +Link: https://patch.msgid.link/20250522232339.1525671-13-helgaas@kernel.org +(cherry picked from commit c8f6791e33a7757025285db26f3b382cdcb7f7cd) +--- + drivers/pci/pci.h | 1 + + drivers/pci/pcie/aer.c | 18 +++++++++--------- + drivers/pci/pcie/dpc.c | 1 + + 3 files changed, 11 insertions(+), 9 deletions(-) + +diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h +index 01234567..89abcdef 100644 +--- a/drivers/pci/pci.h ++++ b/drivers/pci/pci.h +@@ -374,6 +374,7 @@ static inline bool pci_dev_is_added(const struct pci_dev *dev) + struct aer_err_info { + struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES]; + int error_dev_num; ++ const char *level; /* printk level */ + + unsigned int id:16; + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -684,16 +684,14 @@ static void __aer_print_error(struct pci_dev *dev, + { + const char **strings; + unsigned long status = info->status & ~info->mask; +- const char *level, *errmsg; ++ const char *level = info->level; ++ const char *errmsg; + int i; + +- if (info->severity == AER_CORRECTABLE) { ++ if (info->severity == AER_CORRECTABLE) + strings = aer_correctable_error_string; +- level = KERN_WARNING; +- } else { ++ else + strings = aer_uncorrectable_error_string; +- level = KERN_ERR; +- } + + for_each_set_bit(i, &status, 32) { + errmsg = strings[i]; +@@ -709,7 +707,7 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) + { + int layer, agent; + int id = ((dev->bus->number << 8) | dev->devfn); +- const char *level; ++ const char *level = info->level; + + pci_dev_aer_stats_incr(dev, info); + trace_aer_event(pci_name(dev), (info->status & ~info->mask), +@@ -724,8 +722,6 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) + layer = AER_GET_LAYER_ERROR(info->severity, info->status); + agent = AER_GET_AGENT(info->severity, info->status); + +- level = (info->severity == AER_CORRECTABLE) ? KERN_WARNING : KERN_ERR; +- + pci_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n", + aer_error_severity_string[info->severity], + aer_error_layer[layer], aer_agent_string[agent]); +@@ -782,9 +778,11 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + if (aer_severity == AER_CORRECTABLE) { + status = aer->cor_status; + mask = aer->cor_mask; ++ info.level = KERN_WARNING; + } else { + status = aer->uncor_status; + mask = aer->uncor_mask; ++ info.level = KERN_ERR; + tlp_header_valid = status & AER_LOG_TLP_MASKS; + } + +@@ -1147,6 +1145,7 @@ static void aer_isr_one_error(struct aer_rpc *rpc, + if (e_src->status & PCI_ERR_ROOT_COR_RCV) { + e_info.id = ERR_COR_ID(e_src->id); + e_info.severity = AER_CORRECTABLE; ++ e_info.level = KERN_WARNING; + + if (e_src->status & PCI_ERR_ROOT_MULTI_COR_RCV) + e_info.multi_error_valid = 1; +@@ -1160,6 +1159,7 @@ static void aer_isr_one_error(struct aer_rpc *rpc, + + if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { + e_info.id = ERR_UNCOR_ID(e_src->id); ++ e_info.level = KERN_ERR; + + if (e_src->status & PCI_ERR_ROOT_FATAL_RCV) + e_info.severity = AER_FATAL; +diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/dpc.c ++++ b/drivers/pci/pcie/dpc.c +@@ -257,6 +257,7 @@ static int dpc_get_aer_uncorrect_severity(struct pci_dev *dev, + else + info->severity = AER_NONFATAL; + ++ info->level = KERN_ERR; + return 1; + } + +-- +2.47.0 + diff --git a/patches-sonic/0006-PCI-AER-Convert-aer_get_device_error_info-aer_print_.patch b/patches-sonic/0006-PCI-AER-Convert-aer_get_device_error_info-aer_print_.patch new file mode 100644 index 000000000..a9d0b6e80 --- /dev/null +++ b/patches-sonic/0006-PCI-AER-Convert-aer_get_device_error_info-aer_print_.patch @@ -0,0 +1,150 @@ +From bf2793977c116b8125fab34b1ce9136eaa057e7e Mon Sep 17 00:00:00 2001 +From: Bjorn Helgaas +Date: Thu, 22 May 2025 18:21:22 -0500 +Subject: [PATCH 6/8] PCI/AER: Convert aer_get_device_error_info(), + aer_print_error() to index +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Previously aer_get_device_error_info() and aer_print_error() took a pointer +to struct aer_err_info and a pointer to a pci_dev. Typically the pci_dev +was one of the elements of the aer_err_info.dev[] array (DPC was an +exception, where the dev[] array was unused). + +Convert aer_get_device_error_info() and aer_print_error() to take an index +into the aer_err_info.dev[] array instead. A future patch will add +per-device ratelimit information, so the index makes it convenient to find +the ratelimit associated with the device. + +To accommodate DPC, set info->dev[0] to the DPC port before using these +interfaces. + +Signed-off-by: Bjorn Helgaas +Reviewed-by: Ilpo Järvinen +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-17-helgaas@kernel.org +(cherry picked from commit 94bc15c3484aa1dbbd01aee8f9eaa5dc347a01a8) +--- + drivers/pci/pci.h | 5 +++-- + drivers/pci/pcie/aer.c | 33 +++++++++++++++++++++++---------- + drivers/pci/pcie/dpc.c | 8 ++++++-- + 3 files changed, 32 insertions(+), 14 deletions(-) + +diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h +index 01234567..89abcdef 100644 +--- a/drivers/pci/pci.h ++++ b/drivers/pci/pci.h +@@ -391,8 +391,9 @@ struct aer_err_info { + struct aer_header_log_regs tlp; /* TLP Header */ + }; + +-int aer_get_device_error_info(struct pci_dev *dev, struct aer_err_info *info); +-void aer_print_error(struct pci_dev *dev, struct aer_err_info *info); ++int aer_get_device_error_info(struct aer_err_info *info, int i); ++void aer_print_error(struct aer_err_info *info, int i); ++ + #endif /* CONFIG_PCIEAER */ + + #ifdef CONFIG_PCIEPORTBUS +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -703,12 +703,18 @@ static void __aer_print_error(struct pci_dev *dev, + } + } + +-void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) ++void aer_print_error(struct aer_err_info *info, int i) + { +- int layer, agent; +- int id = ((dev->bus->number << 8) | dev->devfn); ++ struct pci_dev *dev; ++ int layer, agent, id; + const char *level = info->level; + ++ if (WARN_ON_ONCE(i >= AER_MAX_MULTI_ERR_DEVICES)) ++ return; ++ ++ dev = info->dev[i]; ++ id = pci_dev_id(dev); ++ + pci_dev_aer_stats_incr(dev, info); + trace_aer_event(pci_name(dev), (info->status & ~info->mask), + info->severity, info->tlp_header_valid, &info->tlp); +@@ -1049,19 +1055,26 @@ EXPORT_SYMBOL_GPL(aer_recover_queue); + + /** + * aer_get_device_error_info - read error status from dev and store it to info +- * @dev: pointer to the device expected to have a error record + * @info: pointer to structure to store the error record ++ * @i: index into info->dev[] + * + * Return 1 on success, 0 on error. + * + * Note that @info is reused among all error devices. Clear fields properly. + */ +-int aer_get_device_error_info(struct pci_dev *dev, struct aer_err_info *info) ++int aer_get_device_error_info(struct aer_err_info *info, int i) + { +- int type = pci_pcie_type(dev); +- int aer = dev->aer_cap; ++ struct pci_dev *dev; ++ int type, aer; + int temp; + ++ if (i >= AER_MAX_MULTI_ERR_DEVICES) ++ return 0; ++ ++ dev = info->dev[i]; ++ aer = dev->aer_cap; ++ type = pci_pcie_type(dev); ++ + /* Must reset in this function */ + info->status = 0; + info->tlp_header_valid = 0; +@@ -1116,11 +1129,11 @@ static inline void aer_process_err_devices(struct aer_err_info *e_info) + + /* Report all before handle them, not to lost records by reset etc. */ + for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { +- if (aer_get_device_error_info(e_info->dev[i], e_info)) +- aer_print_error(e_info->dev[i], e_info); ++ if (aer_get_device_error_info(e_info, i)) ++ aer_print_error(e_info, i); + } + for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { +- if (aer_get_device_error_info(e_info->dev[i], e_info)) ++ if (aer_get_device_error_info(e_info, i)) + handle_error_source(e_info->dev[i], e_info); + } + } +diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/dpc.c ++++ b/drivers/pci/pcie/dpc.c +@@ -258,6 +258,10 @@ static int dpc_get_aer_uncorrect_severity(struct pci_dev *dev, + info->severity = AER_NONFATAL; + + info->level = KERN_ERR; ++ ++ info->dev[0] = dev; ++ info->error_dev_num = 1; ++ + return 1; + } + +@@ -287,8 +291,8 @@ void dpc_process_error(struct pci_dev *pdev) + dpc_process_rp_pio_error(pdev); + else if (reason == 0 && + dpc_get_aer_uncorrect_severity(pdev, &info) && +- aer_get_device_error_info(pdev, &info)) { +- aer_print_error(pdev, &info); ++ aer_get_device_error_info(&info, 0)) { ++ aer_print_error(&info, 0); + pci_aer_clear_nonfatal_status(pdev); + pci_aer_clear_fatal_status(pdev); + } +-- +2.39.5 + diff --git a/patches-sonic/0007-PCI-AER-Ratelimit-correctable-and-non-fatal-error-lo.patch b/patches-sonic/0007-PCI-AER-Ratelimit-correctable-and-non-fatal-error-lo.patch new file mode 100644 index 000000000..6a8c2f75c --- /dev/null +++ b/patches-sonic/0007-PCI-AER-Ratelimit-correctable-and-non-fatal-error-lo.patch @@ -0,0 +1,221 @@ +From 8a2d949e1349ea20ce77a1a42920f67c39597e19 Mon Sep 17 00:00:00 2001 +From: Jon Pan-Doh +Date: Thu, 22 May 2025 18:21:24 -0500 +Subject: [PATCH 7/8] PCI/AER: Ratelimit correctable and non-fatal error + logging + +Spammy devices can flood kernel logs with AER errors and slow/stall +execution. Add per-device ratelimits for AER correctable and non-fatal +uncorrectable errors that use the kernel defaults (10 per 5s). Logging of +fatal errors is not ratelimited. + +There are two AER logging entry points: + + - aer_print_error() is used by DPC and native AER + + - pci_print_aer() is used by GHES and CXL + +The native AER aer_print_error() case includes a loop that may log details +from multiple devices, which are ratelimited individually. If we log +details for any device, we also log the Error Source ID from the Root Port +or RCEC. + +If no such device details are found, we still log the Error Source from the +ERR_* Message, ratelimited by the Root Port or RCEC that received it. + +The DPC aer_print_error() case is not ratelimited, since this only happens +for fatal errors. + +The CXL pci_print_aer() case is ratelimited by the Error Source device. + +The GHES pci_print_aer() case is via aer_recover_work_func(), which +searches for the Error Source device. If the device is not found, there's +no per-device ratelimit, so we use a system-wide ratelimit that covers all +error types (correctable, non-fatal, and fatal). + +Sargun at Meta reported internally that a flood of AER errors causes RCU +CPU stall warnings and CSD-lock warnings. + +Tested using aer-inject[1]. Sent 11 AER errors. Observed 10 errors logged +while AER stats (cat /sys/bus/pci/devices//aer_dev_correctable) show +true count of 11. + +[1] https://git.kernel.org/pub/scm/linux/kernel/git/gong.chen/aer-inject.git + +[bhelgaas: commit log, factor out trace_aer_event() and aer_print_rp_info() +changes to previous patches, enable Error Source logging if any downstream +detail will be printed, don't ratelimit fatal errors, "aer_report" -> +"aer_info", "cor_log_ratelimit" -> "correctable_ratelimit", +"uncor_log_ratelimit" -> "nonfatal_ratelimit"] + +Reported-by: Sargun Dhillon +Signed-off-by: Jon Pan-Doh +Signed-off-by: Bjorn Helgaas +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-19-helgaas@kernel.org +(cherry picked from commit a57f2bfb4a5863f83087867c0e671f2418212d23) +--- + drivers/pci/pci.h | 4 +++- + drivers/pci/pcie/aer.c | 47 +++++++++++++++++++++++++++++++++++++++--- + 2 files changed, 47 insertions(+), 4 deletions(-) + +diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h +index 01234567..89abcdef 100644 +--- a/drivers/pci/pci.h ++++ b/drivers/pci/pci.h +@@ -373,13 +373,15 @@ static inline bool pci_dev_is_added(const struct pci_dev *dev) + + struct aer_err_info { + struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES]; ++ int ratelimit_print[AER_MAX_MULTI_ERR_DEVICES]; + int error_dev_num; + const char *level; /* printk level */ + + unsigned int id:16; + + unsigned int severity:2; /* 0:NONFATAL | 1:FATAL | 2:COR */ +- unsigned int __pad1:5; ++ unsigned int root_ratelimit_print:1; /* 0=skip, 1=print */ ++ unsigned int __pad1:4; + unsigned int multi_error_valid:1; + + unsigned int first_error:5; +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -27,6 +27,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -83,6 +84,10 @@ struct aer_stats { + u64 rootport_total_cor_errs; + u64 rootport_total_fatal_errs; + u64 rootport_total_nonfatal_errs; ++ ++ /* Ratelimits for errors */ ++ struct ratelimit_state correctable_ratelimit; ++ struct ratelimit_state nonfatal_ratelimit; + }; + + #define AER_LOG_TLP_MASKS (PCI_ERR_UNC_POISON_TLP| \ +@@ -382,6 +387,11 @@ void pci_aer_init(struct pci_dev *dev) + + dev->aer_stats = kzalloc(sizeof(struct aer_stats), GFP_KERNEL); + ++ ratelimit_state_init(&dev->aer_stats->correctable_ratelimit, ++ DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); ++ ratelimit_state_init(&dev->aer_stats->nonfatal_ratelimit, ++ DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); ++ + /* + * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER, + * PCI_ERR_COR_MASK, and PCI_ERR_CAP. Root and Root Complex Event +@@ -679,6 +689,18 @@ static void __print_tlp_header(struct pci_dev *dev, + t->dw0, t->dw1, t->dw2, t->dw3); + } + ++static int aer_ratelimit(struct pci_dev *dev, unsigned int severity) ++{ ++ switch (severity) { ++ case AER_NONFATAL: ++ return __ratelimit(&dev->aer_stats->nonfatal_ratelimit); ++ case AER_CORRECTABLE: ++ return __ratelimit(&dev->aer_stats->correctable_ratelimit); ++ default: ++ return 1; /* Don't ratelimit fatal errors */ ++ } ++} ++ + static void __aer_print_error(struct pci_dev *dev, + struct aer_err_info *info) + { +@@ -720,6 +742,9 @@ void aer_print_error(struct aer_err_info *info, int i) + trace_aer_event(pci_name(dev), (info->status & ~info->mask), + info->severity, info->tlp_header_valid, &info->tlp); + ++ if (!info->ratelimit_print[i]) ++ return; ++ + if (!info->status) { + pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n", + aer_error_severity_string[info->severity]); +@@ -800,6 +825,9 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + trace_aer_event(pci_name(dev), (status & ~mask), + aer_severity, tlp_header_valid, &aer->header_log); + ++ if (!aer_ratelimit(dev, info.severity)) ++ return; ++ + layer = AER_GET_LAYER_ERROR(aer_severity, status); + agent = AER_GET_AGENT(aer_severity, status); + +@@ -832,6 +860,18 @@ static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev) + e_info->dev[i] = pci_dev_get(dev); + e_info->error_dev_num++; + ++ /* ++ * Ratelimit AER log messages. "dev" is either the source ++ * identified by the root's Error Source ID or it has an unmasked ++ * error logged in its own AER Capability. Messages are emitted ++ * when "ratelimit_print[i]" is non-zero. If we will print detail ++ * for a downstream device, make sure we print the Error Source ID ++ * from the root as well. ++ */ ++ if (aer_ratelimit(dev, e_info->severity)) { ++ e_info->ratelimit_print[i] = 1; ++ e_info->root_ratelimit_print = 1; ++ } + return 0; + } + +@@ -1009,9 +1049,10 @@ static void aer_recover_work_func(struct work_struct *work) + pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus, + entry.devfn); + if (!pdev) { +- pr_err("no pci_dev for %04x:%02x:%02x.%x\n", +- entry.domain, entry.bus, +- PCI_SLOT(entry.devfn), PCI_FUNC(entry.devfn)); ++ pr_err_ratelimited("%04x:%02x:%02x.%x: no pci_dev found\n", ++ entry.domain, entry.bus, ++ PCI_SLOT(entry.devfn), ++ PCI_FUNC(entry.devfn)); + continue; + } + cper_print_aer(pdev, entry.severity, entry.regs); +@@ -1147,9 +1147,26 @@ static inline void aer_process_err_devices(struct aer_err_info *e_info) + static void aer_isr_one_error_type(struct pci_dev *root, + struct aer_err_info *info) + { +- aer_print_port_info(root, info); ++ bool found; + +- if (find_source_device(root, info)) ++ found = find_source_device(root, info); ++ ++ /* ++ * If we're going to log error messages, we've already set ++ * "info->root_ratelimit_print" and "info->ratelimit_print[i]" to ++ * non-zero (which enables printing) because this is either an ++ * ERR_FATAL or we found a device with an error logged in its AER ++ * Capability. ++ * ++ * If we didn't find the Error Source device, at least log the ++ * Requester ID from the ERR_* Message received by the Root Port or ++ * RCEC, ratelimited by the RP or RCEC. ++ */ ++ if (info->root_ratelimit_print || ++ (!found && aer_ratelimit(root, info->severity))) ++ aer_print_port_info(root, info); ++ ++ if (found) + aer_process_err_devices(info); + } + +-- +2.47.0 + diff --git a/patches-sonic/0008-PCI-AER-Add-sysfs-attributes-for-log-ratelimits.patch b/patches-sonic/0008-PCI-AER-Add-sysfs-attributes-for-log-ratelimits.patch new file mode 100644 index 000000000..49060f895 --- /dev/null +++ b/patches-sonic/0008-PCI-AER-Add-sysfs-attributes-for-log-ratelimits.patch @@ -0,0 +1,272 @@ +From c687a25894edec70b2e663447704c10ba2f58a87 Mon Sep 17 00:00:00 2001 +From: Jon Pan-Doh +Date: Thu, 22 May 2025 18:21:26 -0500 +Subject: [PATCH 8/8] PCI/AER: Add sysfs attributes for log ratelimits + +Allow userspace to read/write log ratelimits per device (including +enable/disable). Create aer/ sysfs directory to store them and any +future AER configs. + +The new sysfs files are: + + /sys/bus/pci/devices/*/aer/correctable_ratelimit_burst + /sys/bus/pci/devices/*/aer/correctable_ratelimit_interval_ms + /sys/bus/pci/devices/*/aer/nonfatal_ratelimit_burst + /sys/bus/pci/devices/*/aer/nonfatal_ratelimit_interval_ms + +The default values are ratelimit_burst=10, ratelimit_interval_ms=5000, so +if we try to emit more than 10 messages in a 5 second period, some are +suppressed. + +Update AER sysfs ABI filename to reflect the broader scope of AER sysfs +attributes (e.g. stats and ratelimits). + + Documentation/ABI/testing/sysfs-bus-pci-devices-aer_stats -> + sysfs-bus-pci-devices-aer + +Tested using aer-inject[1]. Configured correctable log ratelimit to 5. +Sent 6 AER errors. Observed 5 errors logged while AER stats +(cat /sys/bus/pci/devices//aer_dev_correctable) shows 6. + +Disabled ratelimiting and sent 6 more AER errors. Observed all 6 errors +logged and accounted in AER stats (12 total errors). + +[1] https://git.kernel.org/pub/scm/linux/kernel/git/gong.chen/aer-inject.git + +[bhelgaas: note fatal errors are not ratelimited, "aer_report" -> +"aer_info", replace ratelimit_log_enable toggle with *_ratelimit_interval_ms] + +Signed-off-by: Karolina Stolarek +Signed-off-by: Jon Pan-Doh +Signed-off-by: Bjorn Helgaas +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-21-helgaas@kernel.org +(cherry picked from commit b4fe7398def6df344442b884d2288f80205cbd2d) +--- + ...es-aer_stats => sysfs-bus-pci-devices-aer} | 44 ++++++++ + Documentation/PCI/pcieaer-howto.rst | 5 +- + drivers/pci/pci-sysfs.c | 1 + + drivers/pci/pci.h | 1 + + drivers/pci/pcie/aer.c | 105 ++++++++++++++++++ + 5 files changed, 155 insertions(+), 1 deletion(-) + rename Documentation/ABI/testing/{sysfs-bus-pci-devices-aer_stats => sysfs-bus-pci-devices-aer} (72%) + +diff --git a/Documentation/ABI/testing/sysfs-bus-pci-devices-aer_stats b/Documentation/ABI/testing/sysfs-bus-pci-devices-aer +similarity index 72% +rename from Documentation/ABI/testing/sysfs-bus-pci-devices-aer_stats +rename to Documentation/ABI/testing/sysfs-bus-pci-devices-aer +index 01234567..89abcdef 100644 +--- a/Documentation/ABI/testing/sysfs-bus-pci-devices-aer_stats ++++ b/Documentation/ABI/testing/sysfs-bus-pci-devices-aer +@@ -117,3 +117,47 @@ Date: July 2018 + KernelVersion: 4.19.0 + Contact: linux-pci@vger.kernel.org, rajatja@google.com + Description: Total number of ERR_NONFATAL messages reported to rootport. ++ ++PCIe AER ratelimits ++------------------- ++ ++These attributes show up under all the devices that are AER capable. ++They represent configurable ratelimits of logs per error type. ++ ++See Documentation/PCI/pcieaer-howto.rst for more info on ratelimits. ++ ++What: /sys/bus/pci/devices//aer/correctable_ratelimit_interval_ms ++Date: May 2025 ++KernelVersion: 6.16.0 ++Contact: linux-pci@vger.kernel.org ++Description: Writing 0 disables AER correctable error log ratelimiting. ++ Writing a positive value sets the ratelimit interval in ms. ++ Default is DEFAULT_RATELIMIT_INTERVAL (5000 ms). ++ ++What: /sys/bus/pci/devices//aer/correctable_ratelimit_burst ++Date: May 2025 ++KernelVersion: 6.16.0 ++Contact: linux-pci@vger.kernel.org ++Description: Ratelimit burst for correctable error logs. Writing a value ++ changes the number of errors (burst) allowed per interval ++ before ratelimiting. Reading gets the current ratelimit ++ burst. Default is DEFAULT_RATELIMIT_BURST (10). ++ ++What: /sys/bus/pci/devices//aer/nonfatal_ratelimit_interval_ms ++Date: May 2025 ++KernelVersion: 6.16.0 ++Contact: linux-pci@vger.kernel.org ++Description: Writing 0 disables AER non-fatal uncorrectable error log ++ ratelimiting. Writing a positive value sets the ratelimit ++ interval in ms. Default is DEFAULT_RATELIMIT_INTERVAL ++ (5000 ms). ++ ++What: /sys/bus/pci/devices//aer/nonfatal_ratelimit_burst ++Date: May 2025 ++KernelVersion: 6.16.0 ++Contact: linux-pci@vger.kernel.org ++Description: Ratelimit burst for non-fatal uncorrectable error logs. ++ Writing a value changes the number of errors (burst) ++ allowed per interval before ratelimiting. Reading gets the ++ current ratelimit burst. Default is DEFAULT_RATELIMIT_BURST ++ (10). +diff --git a/Documentation/PCI/pcieaer-howto.rst b/Documentation/PCI/pcieaer-howto.rst +index 01234567..89abcdef 100644 +--- a/Documentation/PCI/pcieaer-howto.rst ++++ b/Documentation/PCI/pcieaer-howto.rst +@@ -86,12 +86,15 @@ In the example, 'Requester ID' means the ID of the device who sends + the error message to root port. Pls. refer to pci express specs for + other fields. + ++Ratelimits are exposed in the form of sysfs attributes and configurable. ++See Documentation/ABI/testing/sysfs-bus-pci-devices-aer. ++ + AER Statistics / Counters + ------------------------- + + When PCIe AER errors are captured, the counters / statistics are also exposed + in the form of sysfs attributes which are documented at +-Documentation/ABI/testing/sysfs-bus-pci-devices-aer_stats ++Documentation/ABI/testing/sysfs-bus-pci-devices-aer. + + Developer Guide + =============== +diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pci-sysfs.c ++++ b/drivers/pci/pci-sysfs.c +@@ -1670,6 +1670,7 @@ static const struct attribute_group *pci_dev_attr_groups[] = { + &pcie_dev_attr_group, + #ifdef CONFIG_PCIEAER + &aer_stats_attr_group, ++ &aer_attr_group, + #endif + #ifdef CONFIG_PCIEASPM + &aspm_ctrl_attr_group, +diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h +index 01234567..89abcdef 100644 +--- a/drivers/pci/pci.h ++++ b/drivers/pci/pci.h +@@ -673,6 +673,7 @@ void pci_no_aer(void); + void pci_aer_init(struct pci_dev *dev); + void pci_aer_exit(struct pci_dev *dev); + extern const struct attribute_group aer_stats_attr_group; ++extern const struct attribute_group aer_attr_group; + void pci_aer_clear_fatal_status(struct pci_dev *dev); + int pci_aer_clear_status(struct pci_dev *dev); + int pci_aer_raw_clear_status(struct pci_dev *dev); +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -630,6 +630,111 @@ const struct attribute_group aer_stats_attr_group = { + .is_visible = aer_stats_attrs_are_visible, + }; + ++/* ++ * Ratelimit interval ++ * <=0: disabled with ratelimit.interval = 0 ++ * >0: enabled with ratelimit.interval in ms ++ */ ++#define aer_ratelimit_interval_attr(name, ratelimit) \ ++ static ssize_t \ ++ name##_show(struct device *dev, struct device_attribute *attr, \ ++ char *buf) \ ++ { \ ++ struct pci_dev *pdev = to_pci_dev(dev); \ ++ \ ++ return sysfs_emit(buf, "%d\n", \ ++ pdev->aer_stats->ratelimit.interval); \ ++ } \ ++ \ ++ static ssize_t \ ++ name##_store(struct device *dev, struct device_attribute *attr, \ ++ const char *buf, size_t count) \ ++ { \ ++ struct pci_dev *pdev = to_pci_dev(dev); \ ++ int interval; \ ++ \ ++ if (!capable(CAP_SYS_ADMIN)) \ ++ return -EPERM; \ ++ \ ++ if (kstrtoint(buf, 0, &interval) < 0) \ ++ return -EINVAL; \ ++ \ ++ if (interval <= 0) \ ++ interval = 0; \ ++ else \ ++ interval = msecs_to_jiffies(interval); \ ++ \ ++ pdev->aer_stats->ratelimit.interval = interval; \ ++ \ ++ return count; \ ++ } \ ++ static DEVICE_ATTR_RW(name); ++ ++#define aer_ratelimit_burst_attr(name, ratelimit) \ ++ static ssize_t \ ++ name##_show(struct device *dev, struct device_attribute *attr, \ ++ char *buf) \ ++ { \ ++ struct pci_dev *pdev = to_pci_dev(dev); \ ++ \ ++ return sysfs_emit(buf, "%d\n", \ ++ pdev->aer_stats->ratelimit.burst); \ ++ } \ ++ \ ++ static ssize_t \ ++ name##_store(struct device *dev, struct device_attribute *attr, \ ++ const char *buf, size_t count) \ ++ { \ ++ struct pci_dev *pdev = to_pci_dev(dev); \ ++ int burst; \ ++ \ ++ if (!capable(CAP_SYS_ADMIN)) \ ++ return -EPERM; \ ++ \ ++ if (kstrtoint(buf, 0, &burst) < 0) \ ++ return -EINVAL; \ ++ \ ++ pdev->aer_stats->ratelimit.burst = burst; \ ++ \ ++ return count; \ ++ } \ ++ static DEVICE_ATTR_RW(name); ++ ++#define aer_ratelimit_attrs(name) \ ++ aer_ratelimit_interval_attr(name##_ratelimit_interval_ms, \ ++ name##_ratelimit) \ ++ aer_ratelimit_burst_attr(name##_ratelimit_burst, \ ++ name##_ratelimit) ++ ++aer_ratelimit_attrs(correctable) ++aer_ratelimit_attrs(nonfatal) ++ ++static struct attribute *aer_attrs[] = { ++ &dev_attr_correctable_ratelimit_interval_ms.attr, ++ &dev_attr_correctable_ratelimit_burst.attr, ++ &dev_attr_nonfatal_ratelimit_interval_ms.attr, ++ &dev_attr_nonfatal_ratelimit_burst.attr, ++ NULL ++}; ++ ++static umode_t aer_attrs_are_visible(struct kobject *kobj, ++ struct attribute *a, int n) ++{ ++ struct device *dev = kobj_to_dev(kobj); ++ struct pci_dev *pdev = to_pci_dev(dev); ++ ++ if (!pdev->aer_stats) ++ return 0; ++ ++ return a->mode; ++} ++ ++const struct attribute_group aer_attr_group = { ++ .name = "aer", ++ .attrs = aer_attrs, ++ .is_visible = aer_attrs_are_visible, ++}; ++ + static void pci_dev_aer_stats_incr(struct pci_dev *pdev, + struct aer_err_info *info) + { +-- +2.47.0 + diff --git a/patches-sonic/fix-aer-ratelimiting-sysfs-output-units.patch b/patches-sonic/fix-aer-ratelimiting-sysfs-output-units.patch new file mode 100644 index 000000000..153b25b88 --- /dev/null +++ b/patches-sonic/fix-aer-ratelimiting-sysfs-output-units.patch @@ -0,0 +1,33 @@ +From 02209e4c38f2471d74e82775272096379c44a68f Mon Sep 17 00:00:00 2001 +From: James Sewart +Date: Mon, 10 Nov 2025 13:45:50 +0000 +Subject: [PATCH 4/4] fix aer ratelimiting sysfs output units + +aer statistics include an interval value that determines the time period over which +to limit error logging. This is stored in jiffies, but the sysfs file configuring it +is supposed to expose milliseconds, the store function converts the input to jiffies +and this change ensures the show implementation converts back to milliseconds + +--- + drivers/pci/pcie/aer.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -642,8 +642,9 @@ const struct attribute_group aer_stats_attr_group = { + { \ + struct pci_dev *pdev = to_pci_dev(dev); \ + \ +- return sysfs_emit(buf, "%d\n", \ +- pdev->aer_stats->ratelimit.interval); \ ++ int interval = pdev->aer_stats->ratelimit.interval; \ ++ interval = jiffies_to_msecs(interval); \ ++ return sysfs_emit(buf, "%d\n", interval); \ + } \ + \ + static ssize_t \ +-- +2.39.5 + diff --git a/patches-sonic/series b/patches-sonic/series index dac5959ef..522f9a9a7 100644 --- a/patches-sonic/series +++ b/patches-sonic/series @@ -205,6 +205,20 @@ cisco-npu-disable-other-bars.patch # https://github.com/sonic-net/sonic-buildimage/issues/20901 #PCI-ASPM-Fix-link-state-exit-during-switch-upstream.patch # Upstreamed +# PCIe AER printk ratelimiting +0001-PCI-AER-Simplify-pci_print_aer.patch +0002-PCI-AER-Update-statistics-before-ratelimiting.patch +0003-PCI-AER-Trace-error-event-before-ratelimiting.patch +0004-PCI-AER-Simplify-add_error_device.patch +0005-PCI-AER-Check-log-level-once-and-remember-it.patch +0006-PCI-AER-Convert-aer_get_device_error_info-aer_print_.patch +0001-PCI-AER-Factor-COR-UNCOR-error-handling-out-from-aer.patch +0007-PCI-AER-Ratelimit-correctable-and-non-fatal-error-lo.patch +0008-PCI-AER-Add-sysfs-attributes-for-log-ratelimits.patch +fix-aer-ratelimiting-sysfs-output-units.patch +0001-PCI-AER-Fix-NULL-pointer-access-by-aer_info.patch +0002-PCI-AER-Avoid-NULL-pointer-dereference-in-aer_rateli.patch + # # ############################################################