From 31693fbbfa212bc48eed1b0b20935d3eeb81180c Mon Sep 17 00:00:00 2001 From: Ranjan Kumar Date: Fri, 20 Mar 2026 14:33:24 +0530 Subject: [PATCH 1/3] scsi: mpi3mr: Reset controller on invalid I/O completion Operational replies without a valid scsi_cmnd indicate an invalid I/O completion and a potentially inconsistent controller state. Track this condition and allow the watchdog to trigger a soft reset to safely recover. Signed-off-by: Ranjan Kumar Link: https://patch.msgid.link/20260320090326.47544-2-ranjan.kumar@broadcom.com Signed-off-by: Martin K. Petersen --- drivers/scsi/mpi3mr/mpi3mr.h | 3 +++ drivers/scsi/mpi3mr/mpi3mr_fw.c | 7 +++++++ drivers/scsi/mpi3mr/mpi3mr_os.c | 11 +++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/drivers/scsi/mpi3mr/mpi3mr.h b/drivers/scsi/mpi3mr/mpi3mr.h index 6e962092577d..da141c185eef 100644 --- a/drivers/scsi/mpi3mr/mpi3mr.h +++ b/drivers/scsi/mpi3mr/mpi3mr.h @@ -323,6 +323,7 @@ enum mpi3mr_reset_reason { MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT = 29, MPI3MR_RESET_FROM_SAS_TRANSPORT_TIMEOUT = 30, MPI3MR_RESET_FROM_TRIGGER = 31, + MPI3MR_RESET_FROM_INVALID_COMPLETION = 32, }; #define MPI3MR_RESET_REASON_OSTYPE_LINUX 1 @@ -1183,6 +1184,7 @@ struct scmd_priv { * @num_tb_segs: Number of Segments in Trace buffer * @trace_buf_pool: DMA pool for Segmented trace buffer segments * @trace_buf: Trace buffer segments memory descriptor + * @invalid_io_comp: Invalid IO completion */ struct mpi3mr_ioc { struct list_head list; @@ -1394,6 +1396,7 @@ struct mpi3mr_ioc { u32 num_tb_segs; struct dma_pool *trace_buf_pool; struct segments *trace_buf; + u8 invalid_io_comp; }; diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c index 81150bef1145..b1cd7e4cf40e 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_fw.c +++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c @@ -996,6 +996,7 @@ static const struct { { MPI3MR_RESET_FROM_FIRMWARE, "firmware asynchronous reset" }, { MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT, "configuration request timeout"}, { MPI3MR_RESET_FROM_SAS_TRANSPORT_TIMEOUT, "timeout of a SAS transport layer request" }, + { MPI3MR_RESET_FROM_INVALID_COMPLETION, "invalid cmd completion" }, }; /** @@ -2879,6 +2880,11 @@ static void mpi3mr_watchdog_work(struct work_struct *work) return; } + if (mrioc->invalid_io_comp) { + mpi3mr_soft_reset_handler(mrioc, MPI3MR_RESET_FROM_INVALID_COMPLETION, 1); + return; + } + if (atomic_read(&mrioc->admin_pend_isr)) { ioc_err(mrioc, "Unprocessed admin ISR instance found\n" "flush admin replies\n"); @@ -5644,6 +5650,7 @@ int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc, ssleep(MPI3MR_RESET_TOPOLOGY_SETTLE_TIME); out: + mrioc->invalid_io_comp = 0; if (!retval) { mrioc->diagsave_timeout = 0; mrioc->reset_in_progress = 0; diff --git a/drivers/scsi/mpi3mr/mpi3mr_os.c b/drivers/scsi/mpi3mr/mpi3mr_os.c index 90f8b9d1c2ac..402d1f35d214 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_os.c +++ b/drivers/scsi/mpi3mr/mpi3mr_os.c @@ -3459,8 +3459,15 @@ void mpi3mr_process_op_reply_desc(struct mpi3mr_ioc *mrioc, } scmd = mpi3mr_scmd_from_host_tag(mrioc, host_tag, qidx); if (!scmd) { - panic("%s: Cannot Identify scmd for host_tag 0x%x\n", - mrioc->name, host_tag); + ioc_err(mrioc, "Cannot Identify scmd for host_tag 0x%x", host_tag); + ioc_err(mrioc, + "reply_desc_type(%d) host_tag(%d(0x%04x)): qid(%d): command issued to\n" + "handle(0x%04x) returned with ioc_status(0x%04x), log_info(0x%08x),\n" + "scsi_state(0x%02x), scsi_status(0x%02x), xfer_count(%d), resp_data(0x%08x)\n", + reply_desc_type, host_tag, host_tag, qidx+1, dev_handle, ioc_status, + ioc_loginfo, scsi_state, scsi_status, xfer_count, + resp_data); + mrioc->invalid_io_comp = 1; goto out; } priv = scsi_cmd_priv(scmd); From 9d660e482071bf0bb51f8fe3937eec7448bc8f6a Mon Sep 17 00:00:00 2001 From: Ranjan Kumar Date: Fri, 20 Mar 2026 14:33:25 +0530 Subject: [PATCH 2/3] scsi: mpi3mr: Add queue-full tracking for operational request queues Track queue-full conditions on operational request queues in the driver. Record the last host tag returned to the SCSI mid-layer and count I/Os affected by queue-full conditions. Signed-off-by: Ranjan Kumar Link: https://patch.msgid.link/20260320090326.47544-3-ranjan.kumar@broadcom.com Signed-off-by: Martin K. Petersen --- drivers/scsi/mpi3mr/mpi3mr.h | 12 ++++++++++++ drivers/scsi/mpi3mr/mpi3mr_fw.c | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/drivers/scsi/mpi3mr/mpi3mr.h b/drivers/scsi/mpi3mr/mpi3mr.h index da141c185eef..631a48f7425d 100644 --- a/drivers/scsi/mpi3mr/mpi3mr.h +++ b/drivers/scsi/mpi3mr/mpi3mr.h @@ -429,6 +429,14 @@ struct segments { * @q_segments: Segment descriptor pointer * @q_segment_list: Segment list base virtual address * @q_segment_list_dma: Segment list base DMA address + * @last_full_host_tag: Hosttag of last IO returned to SML + * due to queue full + * @qfull_io_count: Number of IOs returned back to SML + * due to queue full + * @qfull_instances: Total queue full occurrences.One occurrence + * starts with queue full detection and ends + * with queue full breaks. + * */ struct op_req_qinfo { u16 ci; @@ -442,6 +450,10 @@ struct op_req_qinfo { struct segments *q_segments; void *q_segment_list; dma_addr_t q_segment_list_dma; + u16 last_full_host_tag; + u64 qfull_io_count; + u32 qfull_instances; + }; /** diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c index b1cd7e4cf40e..54debf16c2e6 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_fw.c +++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c @@ -2362,6 +2362,9 @@ static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx, op_req_q->ci = 0; op_req_q->pi = 0; op_req_q->reply_qid = reply_qid; + op_req_q->last_full_host_tag = MPI3MR_HOSTTAG_INVALID; + op_req_q->qfull_io_count = 0; + op_req_q->qfull_instances = 0; spin_lock_init(&op_req_q->q_lock); if (!op_req_q->q_segments) { @@ -2548,6 +2551,8 @@ int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc, u16 req_sz = mrioc->facts.op_req_sz; struct segments *segments = op_req_q->q_segments; struct op_reply_qinfo *op_reply_q = NULL; + struct mpi3_scsi_io_request *scsiio_req = + (struct mpi3_scsi_io_request *)req; reply_qidx = op_req_q->reply_qid - 1; op_reply_q = mrioc->op_reply_qinfo + reply_qidx; @@ -2565,11 +2570,21 @@ int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc, mpi3mr_process_op_reply_q(mrioc, mrioc->intr_info[midx].op_reply_q); if (mpi3mr_check_req_qfull(op_req_q)) { + + if (op_req_q->last_full_host_tag == + MPI3MR_HOSTTAG_INVALID) + op_req_q->qfull_instances++; + + op_req_q->last_full_host_tag = scsiio_req->host_tag; + op_req_q->qfull_io_count++; retval = -EAGAIN; goto out; } } + if (op_req_q->last_full_host_tag != MPI3MR_HOSTTAG_INVALID) + op_req_q->last_full_host_tag = MPI3MR_HOSTTAG_INVALID; + if (mrioc->reset_in_progress) { ioc_err(mrioc, "OpReqQ submit reset in progress\n"); retval = -EAGAIN; @@ -4827,6 +4842,7 @@ void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc) mrioc->req_qinfo[i].qid = 0; mrioc->req_qinfo[i].reply_qid = 0; spin_lock_init(&mrioc->req_qinfo[i].q_lock); + mrioc->req_qinfo[i].last_full_host_tag = 0; mpi3mr_memset_op_req_q_buffers(mrioc, i); } From 02ff1d2bcf2d67bfa08ce7135bd3b33d1fffca61 Mon Sep 17 00:00:00 2001 From: Ranjan Kumar Date: Fri, 20 Mar 2026 14:33:26 +0530 Subject: [PATCH 3/3] scsi: mpi3mr: Add retry mechanism for IOC shutdown with timeout reset Enhance the IOC shutdown process to handle transient failures during controller cleanup. Add retry logic with configurable maximum retry count (MPI3MR_MAX_SHUTDOWN_RETRY_COUNT) and proper timeout management that resets on each retry attempt. This ensures shutdown can recover from temporary issues without failing completely. Signed-off-by: Ranjan Kumar Link: https://patch.msgid.link/20260320090326.47544-4-ranjan.kumar@broadcom.com Signed-off-by: Martin K. Petersen --- drivers/scsi/mpi3mr/mpi3mr.h | 1 + drivers/scsi/mpi3mr/mpi3mr_fw.c | 32 ++++++++++++++++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/drivers/scsi/mpi3mr/mpi3mr.h b/drivers/scsi/mpi3mr/mpi3mr.h index 631a48f7425d..c25525fe0671 100644 --- a/drivers/scsi/mpi3mr/mpi3mr.h +++ b/drivers/scsi/mpi3mr/mpi3mr.h @@ -159,6 +159,7 @@ extern atomic64_t event_counter; /* Controller Reset related definitions */ #define MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT 5 #define MPI3MR_MAX_RESET_RETRY_COUNT 3 +#define MPI3MR_MAX_SHUTDOWN_RETRY_COUNT 2 /* ResponseCode definitions */ #define MPI3MR_RI_MASK_RESPCODE (0x000000FF) diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c index 54debf16c2e6..01042eaf0dff 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_fw.c +++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c @@ -5058,9 +5058,9 @@ void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc) */ static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc) { - u32 ioc_config, ioc_status; - u8 retval = 1; - u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10; + u32 ioc_config, ioc_status, shutdown_action; + u8 retval = 1, retry = 0; + u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10, timeout_remaining = 0; ioc_info(mrioc, "Issuing shutdown Notification\n"); if (mrioc->unrecoverable) { @@ -5075,14 +5075,16 @@ static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc) return; } + shutdown_action = MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL | + MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN_SEND_REQ; ioc_config = readl(&mrioc->sysif_regs->ioc_configuration); - ioc_config |= MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL; - ioc_config |= MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN_SEND_REQ; + ioc_config |= shutdown_action; writel(ioc_config, &mrioc->sysif_regs->ioc_configuration); if (mrioc->facts.shutdown_timeout) timeout = mrioc->facts.shutdown_timeout * 10; + timeout_remaining = timeout; do { ioc_status = readl(&mrioc->sysif_regs->ioc_status); @@ -5091,8 +5093,26 @@ static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc) retval = 0; break; } + if (mrioc->unrecoverable) + break; + if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) { + mpi3mr_print_fault_info(mrioc); + if (retry >= MPI3MR_MAX_SHUTDOWN_RETRY_COUNT) + break; + if (mpi3mr_issue_reset(mrioc, + MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, + MPI3MR_RESET_FROM_CTLR_CLEANUP)) + break; + ioc_config = + readl(&mrioc->sysif_regs->ioc_configuration); + ioc_config |= shutdown_action; + writel(ioc_config, + &mrioc->sysif_regs->ioc_configuration); + timeout_remaining = timeout; + retry++; + } msleep(100); - } while (--timeout); + } while (--timeout_remaining); ioc_status = readl(&mrioc->sysif_regs->ioc_status); ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);