From 8f5d59b9cfa69beab3b52cca621bb301c5af98aa Mon Sep 17 00:00:00 2001 From: Michael Chan Date: Mon, 5 Oct 2026 13:42:44 -0700 Subject: [PATCH 1/3] bnxt_en: Add bnxt_clear_bars() helper In bnxt_io_slot_reset(), we clear the 6 BAR registers. Add a helper function to do that. The helper will be used again in the next patch. Reviewed-by: Kalesh AP Reviewed-by: Somnath Kotur Reviewed-by: Joe Damato Signed-off-by: Pavan Chebbi Signed-off-by: Michael Chan Reviewed-by: Jacob Keller Link: https://patch.msgid.link/20261005204246.3822563-2-michael.chan@broadcom.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index d7728d0c5b6e..8ce8a82d3453 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -17111,6 +17111,14 @@ void bnxt_print_device_info(struct bnxt *bp) pcie_print_link_status(bp->pdev); } +static void bnxt_clear_bars(struct pci_dev *pdev) +{ + int off; + + for (off = PCI_BASE_ADDRESS_0; off <= PCI_BASE_ADDRESS_5; off += 4) + pci_write_config_dword(pdev, off, 0); +} + static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) { struct bnxt_hw_resc *hw_resc; @@ -17598,7 +17606,6 @@ static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) struct bnxt *bp = netdev_priv(netdev); int retry = 0; int err = 0; - int off; netdev_info(bp->dev, "PCI Slot Reset\n"); @@ -17627,11 +17634,8 @@ static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) * write the BARs to 0 to force restore, in case of fatal error. */ if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, - &bp->state)) { - for (off = PCI_BASE_ADDRESS_0; - off <= PCI_BASE_ADDRESS_5; off += 4) - pci_write_config_dword(bp->pdev, off, 0); - } + &bp->state)) + bnxt_clear_bars(pdev); pci_restore_state(pdev); bnxt_inv_fw_health_reg(bp); From 386f887c8290f0747dd0180e98ec7a82811bc506 Mon Sep 17 00:00:00 2001 From: Michael Chan Date: Mon, 5 Oct 2026 13:42:45 -0700 Subject: [PATCH 2/3] bnxt_en: Fix driver init in kdump kernel Fix and strengthen the FLR sequence when initializing in the kdump kernel. If the NIC is behind a PCIe switch in synthetic (smart) mode, the switch may need to see that the BARs have been initialized before it will pass mem read/write TLPs to the NIC. On a Dell system with a PEX89144 PCIe switch, echo c > /proc/sysrq-trigger will trigger fatal AER without this patch: bnxt_en 0000:67:00.0: enabling device (0000 -> 0002) [Hardware Error]: Hardware error from APEI Generic Hardware Error Source: 5 [Hardware Error]: event severity: recoverable [Hardware Error]: Error 0, type: fatal [Hardware Error]: section_type: PCIe error [Hardware Error]: port_type: 5, upstream switch port ... Add a new bnxt_kdump_reset() to do the expanded FLR sequence in the kdump kernel. We now disable bus master and memory, save the PCI state, do the FLR, clear the BARs, and restore the PCI state. The BARs have to be cleared to ensure that they get re-initialized and visible to the PCIe switch. Since it is the kdump kernel, we make every effort to continue in the best possible way even if pci_save_state() or pcie_flr() returns error. After FLR, we poll for an additional 5 seconds before aborting in case the device is not properly returning CRS. This is similar to the 5-second wait in bnxt_io_slot_reset(). Fixes: 8743db4a9acf ("bnxt_en: Issue PCIe FLR in kdump kernel to cleanup pending DMAs.") Reviewed-by: Kalesh AP Signed-off-by: Pavan Chebbi Signed-off-by: Michael Chan Reviewed-by: Jacob Keller Link: https://patch.msgid.link/20261005204246.3822563-3-michael.chan@broadcom.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 45 ++++++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 8ce8a82d3453..9ea7e172787e 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -17119,6 +17119,43 @@ static void bnxt_clear_bars(struct pci_dev *pdev) pci_write_config_dword(pdev, off, 0); } +/* Clear any pending DMA transactions from crash kernel while loading driver in + * capture kernel. + */ +static int bnxt_kdump_reset(struct pci_dev *pdev) +{ + int rc, i; + u16 cmd; + + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + cmd &= ~(PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY); + pci_write_config_word(pdev, PCI_COMMAND, cmd); + + if (pci_save_state(pdev)) + dev_warn(&pdev->dev, "Failed to save PCI state, PCI restore may be incomplete\n"); + + rc = pcie_flr(pdev); + if (rc) + dev_warn(&pdev->dev, "pcie_flr() failed (rc: %d), trying to continue\n", + rc); + + /* In case device is not returning CRS, wait 5 seconds longer */ + for (i = 0; i < 50; i++) { + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + if (!PCI_POSSIBLE_ERROR(cmd)) + break; + msleep(100); + } + if (PCI_POSSIBLE_ERROR(cmd)) { + dev_err(&pdev->dev, "PCI config space inaccessible after FLR, aborting\n"); + return -ENODEV; + } + + bnxt_clear_bars(pdev); + pci_restore_state(pdev); + return 0; +} + static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) { struct bnxt_hw_resc *hw_resc; @@ -17134,12 +17171,10 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENODEV; } - /* Clear any pending DMA transactions from crash kernel - * while loading driver in capture kernel. - */ if (is_kdump_kernel()) { - pci_clear_master(pdev); - pcie_flr(pdev); + rc = bnxt_kdump_reset(pdev); + if (rc) + return rc; } max_irqs = bnxt_get_max_irq(pdev); From 2e52da27096d9dbc487f303bbc10d44ad837a8c4 Mon Sep 17 00:00:00 2001 From: Pavan Chebbi Date: Mon, 5 Oct 2026 13:42:46 -0700 Subject: [PATCH 3/3] bnxt_en: Re-write the BARs following any type of PCIe errors Currently the driver zeroes the BARs only when fatal PCIe errors are reported so that pci_restore_state() restores it. However firmware handles both fatal and non-fatal errors the same way when it sees the slot reset resulting from the PCI_ERS_RESULT_NEED_RESET return code from the driver. This means that we must re-write the BARs post recovery even during non-fatal errors. Otherwise we will see that every MMIO access returns all-ones and the firmware appears dead. Zero-out the BARs during PCIe error recovery regardless of type of PCIe error, and make the wait after the hot reset unconditional. Disable memory decode and bus mastering before rewriting the BARs so the device doesn't decode a half-updated address, bailing out if config space is still inaccessible. Defer pci_enable_device() until after the BAR rewrite and restore, so the device isn't re-enabled while its BARs are still being rewritten, then re-enable the device and re-assert bus mastering. Guard the same Command register cleanup on the re-enable failure path against an inaccessible device. Skip re-enabling the device in bnxt_io_slot_reset() if it is already enabled, so enable_cnt does not go unbalanced. A concurrent bnxt_fw_reset_task() can also be re-enabling the same device in its ENABLE_DEV state, so guard that call the same way. Fixes: f75d9a0aa967 ("bnxt_en: Re-write PCI BARs after PCI fatal error.") Reviewed-by: Kalesh AP Reviewed-by: Scott Branden Signed-off-by: Pavan Chebbi Signed-off-by: Michael Chan Reviewed-by: Jacob Keller Link: https://patch.msgid.link/20261005204246.3822563-4-michael.chan@broadcom.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 117 ++++++++++++---------- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 1 - 2 files changed, 65 insertions(+), 53 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 9ea7e172787e..5bd817479d64 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -15547,7 +15547,7 @@ static void bnxt_fw_reset_task(struct work_struct *work) if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) && !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state)) bnxt_dl_remote_reload(bp); - if (pci_enable_device(bp->pdev)) { + if (!pci_is_enabled(bp->pdev) && pci_enable_device(bp->pdev)) { netdev_err(bp->dev, "Cannot re-enable PCI device\n"); rc = -ENODEV; goto fw_reset_abort; @@ -17608,10 +17608,8 @@ static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, * so we disable bus master to prevent any potential bad DMAs before * freeing kernel memory. */ - if (state == pci_channel_io_frozen) { - set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state); + if (state == pci_channel_io_frozen) bnxt_fw_fatal_close(bp); - } if (netif_running(netdev)) __bnxt_close_nic(bp, true, true); @@ -17641,64 +17639,79 @@ static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) struct bnxt *bp = netdev_priv(netdev); int retry = 0; int err = 0; + u16 cmd; netdev_info(bp->dev, "PCI Slot Reset\n"); - if (test_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state)) { - /* After DPC, the chip should return CRS when the vendor ID - * config register is read until it is ready. On all chips, - * this is not happening reliably so add a 5-second delay as a - * workaround. - */ - msleep(5000); - } + /* After a PCIe hot reset, the chip should return CRS when the + * vendor ID config register is read until it is ready. On all + * chips, this is not happening reliably so add a 5-second delay + * as a workaround. + */ + msleep(5000); netdev_lock(netdev); - if (pci_enable_device(pdev)) { + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + if (PCI_POSSIBLE_ERROR(cmd)) { + dev_err(&pdev->dev, + "PCI config space inaccessible after reset\n"); + goto reset_exit; + } + + /* Upon PCIe error, our device internal logic that latches to + * BAR value is getting reset and will restore only upon + * rewriting the BARs. + * + * As pci_restore_state() does not re-write the BARs if the + * value is same as saved value earlier, driver needs to + * write the BARs to 0 to force restore. + */ + pci_clear_master(pdev); + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + cmd &= ~PCI_COMMAND_MEMORY; + pci_write_config_word(pdev, PCI_COMMAND, cmd); + + bnxt_clear_bars(pdev); + pci_restore_state(pdev); + + if (!pci_is_enabled(pdev) && pci_enable_device(pdev)) { dev_err(&pdev->dev, "Cannot re-enable PCI device after reset.\n"); - } else { - pci_set_master(pdev); - /* Upon fatal error, our device internal logic that latches to - * BAR value is getting reset and will restore only upon - * rewriting the BARs. - * - * As pci_restore_state() does not re-write the BARs if the - * value is same as saved value earlier, driver needs to - * write the BARs to 0 to force restore, in case of fatal error. - */ - if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, - &bp->state)) - bnxt_clear_bars(pdev); - pci_restore_state(pdev); - - bnxt_inv_fw_health_reg(bp); - bnxt_try_map_fw_health_reg(bp); - - /* In some PCIe AER scenarios, firmware may take up to - * 10 seconds to become ready in the worst case. - */ - do { - err = bnxt_try_recover_fw(bp); - if (!err) - break; - retry++; - } while (retry < BNXT_FW_SLOT_RESET_RETRY); - - if (err) { - dev_err(&pdev->dev, "Firmware not ready\n"); - goto reset_exit; + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + if (!PCI_POSSIBLE_ERROR(cmd)) { + cmd &= ~(PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY); + pci_write_config_word(pdev, PCI_COMMAND, cmd); } - - err = bnxt_hwrm_func_reset(bp); - if (!err) - result = PCI_ERS_RESULT_RECOVERED; - - /* IRQ will be initialized later in bnxt_io_resume */ - bnxt_ulp_irq_stop(bp); - bnxt_clear_int_mode(bp); + goto reset_exit; } + pci_set_master(pdev); + + bnxt_inv_fw_health_reg(bp); + bnxt_try_map_fw_health_reg(bp); + + /* In some PCIe AER scenarios, firmware may take up to + * 10 seconds to become ready in the worst case. + */ + do { + err = bnxt_try_recover_fw(bp); + if (!err) + break; + retry++; + } while (retry < BNXT_FW_SLOT_RESET_RETRY); + + if (err) { + dev_err(&pdev->dev, "Firmware not ready\n"); + goto reset_exit; + } + + err = bnxt_hwrm_func_reset(bp); + if (!err) + result = PCI_ERS_RESULT_RECOVERED; + + /* IRQ will be initialized later in bnxt_io_resume */ + bnxt_ulp_irq_stop(bp); + bnxt_clear_int_mode(bp); reset_exit: clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index c673b2ce4a0d..41d05efd6307 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -2468,7 +2468,6 @@ struct bnxt { #define BNXT_STATE_ABORT_ERR 5 #define BNXT_STATE_FW_FATAL_COND 6 #define BNXT_STATE_DRV_REGISTERED 7 -#define BNXT_STATE_PCI_CHANNEL_IO_FROZEN 8 #define BNXT_STATE_NAPI_DISABLED 9 #define BNXT_STATE_FW_ACTIVATE 11 #define BNXT_STATE_RECOVER 12