From 02a3178319639f650428164d4d003de4dc03e326 Mon Sep 17 00:00:00 2001 From: Naomi Chu Date: Wed, 9 Sep 2026 17:10:45 +0800 Subject: [PATCH 1/6] scsi: ufs: core: Hold a clock reference across the probe Clock gating becomes possible as soon as ufshcd_init_clk_gating() has run, and from that point on the probe keeps accessing host registers without ever taking a clock reference. This has been safe only because of the state check in __ufshcd_release(): gate_work is not queued unless hba->ufshcd_state is UFSHCD_STATE_OPERATIONAL, and the promotion to that state used to happen after the last register access of the probe, at the end of ufshcd_probe_hba(). That is fragile: it only works while the promotion happens after the register accesses. Commit a390e6677f41 ("scsi: ufs: core: Expand the ufshcd_device_init(hba, true) call") changed that ordering by moving the promotion into ufshcd_init(), which schedules ufshcd_async_scan() afterwards. ufshcd_probe_hba() therefore now runs with the state already promoted, and it accesses host registers without holding a clock reference: - On hosts with UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH it calls ufshcd_hba_stop() and ufshcd_hba_enable() before ufshcd_device_init() sets the state back to UFSHCD_STATE_RESET, and both read REG_CONTROLLER_ENABLE, so gated clocks stall there instead of just losing a write. - It ends with an ufshcd_configure_auto_hibern8() write, which its other callers do take a clock reference for. Take a clock reference as soon as clock gating has been initialised and keep it until the probe is over. It then does not matter who drops a clock reference while the probe is running, and the register accesses of the probe no longer depend on hba->ufshcd_state. The reference is dropped by ufshcd_async_scan() once the scan has finished, or by the new out_release label if the probe fails after it was taken. Fixes: a390e6677f41 ("scsi: ufs: core: Expand the ufshcd_device_init(hba, true) call") Signed-off-by: Naomi Chu Reviewed-by: Peter Wang Link: https://patch.msgid.link/20260909091045.1134956-1-naomi.chu@mediatek.com Signed-off-by: Martin K. Petersen (Oracle) --- drivers/ufs/core/ufshcd.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 54f4e7d7de02..79b6a2d9b522 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -9556,6 +9556,7 @@ static void ufshcd_async_scan(void *data, async_cookie_t cookie) ret = ufshcd_add_lus(hba); out: + ufshcd_release(hba); pm_runtime_put_sync(hba->dev); if (ret) @@ -11278,6 +11279,9 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) ufshcd_init_clk_gating(hba); + /* Released by ufshcd_async_scan(), or by out_release on failure. */ + ufshcd_hold(hba); + ufshcd_init_clk_scaling(hba); /* @@ -11298,7 +11302,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba); if (err) { dev_err(hba->dev, "request irq failed\n"); - goto out_disable; + goto out_release; } else { hba->is_irq_enabled = true; } @@ -11314,7 +11318,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) dev_err(hba->dev, "Host controller enable failed\n"); ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); - goto out_disable; + goto out_release; } INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work); @@ -11328,7 +11332,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) err = ufshcd_add_scsi_host(hba); if (err) - goto out_disable; + goto out_release; /* Hold auto suspend until async scan completes */ pm_runtime_get_sync(dev); @@ -11348,7 +11352,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) err = ufshcd_link_startup(hba); if (err) - goto out_disable; + goto out_release; if (hba->mcq_enabled) ufshcd_config_mcq(hba); @@ -11365,23 +11369,23 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) /* Verify device initialization by sending NOP OUT UPIU */ err = ufshcd_verify_dev_init(hba); if (err) - goto out_disable; + goto out_release; /* Initiate UFS initialization, and waiting until completion */ err = ufshcd_complete_dev_init(hba); if (err) - goto out_disable; + goto out_release; err = ufshcd_device_params_init(hba); if (err) - goto out_disable; + goto out_release; err = ufshcd_post_device_init(hba); initialized: ufshcd_process_probe_result(hba, probe_start, err); if (err) - goto out_disable; + goto out_release; ufs_sysfs_add_nodes(hba->dev); hba->dme_qos_sysfs_handle = sysfs_get_dirent(hba->dev->kobj.sd, @@ -11392,6 +11396,8 @@ initialized: ufshcd_pm_qos_init(hba); return 0; +out_release: + ufshcd_release(hba); out_disable: hba->is_irq_enabled = false; ufshcd_hba_exit(hba); From 85de8b3a5eaa23d153aff4c7c83dddbc949064c9 Mon Sep 17 00:00:00 2001 From: Runyu Xiao Date: Thu, 10 Sep 2026 17:24:05 +0800 Subject: [PATCH 2/6] scsi: qedi: Initialize callback state before registration qedi_get_protocol_tlv_data() can be called asynchronously by QED after qedi_ops->register_ops(). It takes stats_lock and reads ll2_mtu, but __qedi_probe() currently registers the callback before initializing the mutex and assigning the default MTU on the normal probe path. Initialize the callback-visible state before registering qedi_cb_ops. Keep the recovery path from reinitializing state because it reuses the existing qedi context. Fixes: 3cc5746e5ad7 ("scsi: qedi: Initialize the stats mutex lock") Cc: stable@vger.kernel.org Assisted-by: LLM Codex Signed-off-by: Runyu Xiao Link: https://patch.msgid.link/20260910092405.1300129-1-runyu.xiao@seu.edu.cn Signed-off-by: Martin K. Petersen (Oracle) --- drivers/scsi/qedi/qedi_main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/scsi/qedi/qedi_main.c b/drivers/scsi/qedi/qedi_main.c index 227ff7bd1bdc..f394fcbb97e7 100644 --- a/drivers/scsi/qedi/qedi_main.c +++ b/drivers/scsi/qedi/qedi_main.c @@ -2652,8 +2652,6 @@ retry_probe: snprintf(host_buf, sizeof(host_buf), "host_%d", qedi->shost->host_no); qedi_ops->common->set_name(qedi->cdev, host_buf); - qedi_ops->register_ops(qedi->cdev, &qedi_cb_ops, qedi); - memset(¶ms, 0, sizeof(params)); params.mtu = DEF_PATH_MTU + IPV6_HDR_LEN + TCP_HDR_LEN; qedi->ll2_mtu = DEF_PATH_MTU; @@ -2670,6 +2668,7 @@ retry_probe: spin_lock_init(&qedi->task_idx_lock); mutex_init(&qedi->stats_lock); } + qedi_ops->register_ops(qedi->cdev, &qedi_cb_ops, qedi); qedi_ops->ll2->register_cb_ops(qedi->cdev, &qedi_ll2_cb_ops, qedi); qedi_ops->ll2->start(qedi->cdev, ¶ms); From 78e04810542da416d3bead707b60cec773f1524d Mon Sep 17 00:00:00 2001 From: Pengpeng Hou Date: Sun, 20 Sep 2026 11:49:53 +0800 Subject: [PATCH 3/6] scsi: ufs: mediatek: Handle mPHY power-on failures The MediaTek helper enables VA09 before powering on the mPHY, but ignores the PHY result. A failure can leave VA09 enabled and publish mphy_powered_on as true, so a later power-on attempt is skipped. Check phy_power_on(), attempt to unwind VA09 on failure, and leave the state flag unchanged. Propagate the helper result from host initialization as well as its existing resume caller. Keep the original PHY error if the supply rollback also fails, and report that rollback separately. The issue was found by our static-analysis tool. Fixes: 561e3a8726b2 ("scsi: ufs-mediatek: Fix unbalanced clock on/off") Fixes: cf137b3ea49a ("scsi: ufs-mediatek: Support VA09 regulator operations") Assisted-by: gpt 5 Signed-off-by: Pengpeng Hou Reviewed-by: Peter Wang Reviewed-by: Stanley Jhu Link: https://patch.msgid.link/20260920034953.19535-1-hppiscas@163.com Signed-off-by: Martin K. Petersen (Oracle) --- drivers/ufs/host/ufs-mediatek.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c index 814c1b7343b9..12a6d4138e44 100644 --- a/drivers/ufs/host/ufs-mediatek.c +++ b/drivers/ufs/host/ufs-mediatek.c @@ -509,6 +509,7 @@ static int ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on) struct ufs_mtk_host *host = ufshcd_get_variant(hba); struct phy *mphy = host->mphy; struct arm_smccc_res res; + int err; int ret = 0; if (!mphy || !(on ^ host->mphy_powered_on)) @@ -523,7 +524,18 @@ static int ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on) usleep_range(200, 210); ufs_mtk_va09_pwr_ctrl(res, 1); } - phy_power_on(mphy); + ret = phy_power_on(mphy); + if (ret) { + if (ufs_mtk_is_va09_supported(hba)) { + ufs_mtk_va09_pwr_ctrl(res, 0); + err = regulator_disable(host->reg_va09); + if (err) + dev_err(hba->dev, + "failed to unwind va09: %d\n", err); + } + dev_err(hba->dev, "failed to enable mphy: %d\n", ret); + return ret; + } } else { phy_power_off(mphy); if (ufs_mtk_is_va09_supported(hba)) { @@ -1295,7 +1307,9 @@ static int ufs_mtk_init(struct ufs_hba *hba) * * Enable phy clocks specifically here. */ - ufs_mtk_mphy_power_on(hba, true); + err = ufs_mtk_mphy_power_on(hba, true); + if (err) + goto out_variant_clear; if (ufs_mtk_is_rtff_mtcmos(hba)) { /* First Restore here, to avoid backup unexpected value */ From 27c0f718d3b3c99e6d315308e95d987d0382a8a8 Mon Sep 17 00:00:00 2001 From: Ahmed Abdelhaleem Ahmed Date: Tue, 22 Sep 2026 15:46:46 +0000 Subject: [PATCH 4/6] scsi: ch: Do not keep references to data transfer element devices ch_readconfig() looks up the scsi_device of every data transfer element whose SCSI id the changer reports in READ ELEMENT STATUS, and stores it in ch->dt[]. scsi_device_lookup() takes a reference, and nothing ever drops it: ch_destroy() frees the array with kfree(). ch->dt[] is read nowhere else - it only supplies the vendor, model and revision printed in the same loop. Once such a drive is removed, its scsi_device can never be released. It stays on the host's device list at its address, so a new device there is refused by anything that walks the list - target_core_pscsi reports "scsi_device_get() failed for H:C:T:L" - and the low-level driver's module can no longer be unloaded. Only a reboot recovers. It shows with any changer that reports its drives' ids; with the mhvtl virtual library (IBM 3573-TL personality), each create and remove of a library with four drives leaves four references behind, counted by the module's use count in lsmod. With ch not bound the count is unchanged, and with this patch applied it is unchanged too. Drop the reference as soon as the name has been printed, and remove the now unused dt[] array. That also removes the array leaked when ch_probe() fails after ch_readconfig(). The same leak was reported with an RFC patch in 2022, which was not merged. Fixes: daa6eda65a53 ("[SCSI] add scsi changer driver") Cc: stable@vger.kernel.org Link: https://lore.kernel.org/linux-scsi/20220719075442.6215-1-yanghao_ht@163.com/ Signed-off-by: Ahmed Abdelhaleem Ahmed Reviewed-by: Laurence Oberman Fixes: daa6eda65a53 ("[SCSI] add scsi changer driver") Link: https://patch.msgid.link/20260929-ch-dt-leak-v2-1-ddda8d635dca@gmail.com Signed-off-by: Martin K. Petersen (Oracle) --- drivers/scsi/ch.c | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/drivers/scsi/ch.c b/drivers/scsi/ch.c index 4010fdbf813c..c36539c83ed1 100644 --- a/drivers/scsi/ch.c +++ b/drivers/scsi/ch.c @@ -112,7 +112,6 @@ typedef struct { int minor; char name[8]; struct scsi_device *device; - struct scsi_device **dt; /* ptrs to data transfer elements */ u_int firsts[CH_TYPES]; u_int counts[CH_TYPES]; u_int voltags; @@ -363,15 +362,10 @@ ch_readconfig(scsi_changer *ch) vendor_labels[i]); } - /* look up the devices of the data transfer elements */ - ch->dt = kzalloc_objs(*ch->dt, ch->counts[CHET_DT]); - - if (!ch->dt) { - kfree(buffer); - return -ENOMEM; - } - + /* report the devices of the data transfer elements */ for (elem = 0; elem < ch->counts[CHET_DT]; elem++) { + struct scsi_device *sdev; + id = -1; lun = 0; if (elem < CH_DT_MAX && -1 != dt_id[elem]) { @@ -387,10 +381,8 @@ ch_readconfig(scsi_changer *ch) VPRINTK(KERN_INFO, "dt 0x%x: ",elem+ch->firsts[CHET_DT]); if (data[6] & 0x80) { VPRINTK(KERN_CONT, "not this SCSI bus\n"); - ch->dt[elem] = NULL; } else if (0 == (data[6] & 0x30)) { VPRINTK(KERN_CONT, "ID/LUN unknown\n"); - ch->dt[elem] = NULL; } else { id = ch->device->id; lun = 0; @@ -400,18 +392,16 @@ ch_readconfig(scsi_changer *ch) } if (-1 != id) { VPRINTK(KERN_CONT, "ID %i, LUN %i, ",id,lun); - ch->dt[elem] = - scsi_device_lookup(ch->device->host, - ch->device->channel, - id,lun); - if (!ch->dt[elem]) { + sdev = scsi_device_lookup(ch->device->host, + ch->device->channel, + id, lun); + if (!sdev) { /* should not happen */ VPRINTK(KERN_CONT, "Huh? device not found!\n"); } else { VPRINTK(KERN_CONT, "name: %8.8s %16.16s %4.4s\n", - ch->dt[elem]->vendor, - ch->dt[elem]->model, - ch->dt[elem]->rev); + sdev->vendor, sdev->model, sdev->rev); + scsi_device_put(sdev); } } } @@ -575,7 +565,6 @@ static void ch_destroy(struct kref *ref) scsi_changer *ch = container_of(ref, scsi_changer, ref); ch->device = NULL; - kfree(ch->dt); kfree(ch); } From 375f3a5691dd46b5d05b90a8968872ec8b2248c5 Mon Sep 17 00:00:00 2001 From: Stanley Jhu Date: Mon, 28 Sep 2026 11:58:15 +0800 Subject: [PATCH 5/6] scsi: ufs: core: Avoid unsafe MMIO reads in ufshcd_mcq_compl_all_cqes_lock() During MCQ host reset, ufshcd_host_reset_and_restore() stops the host controller via ufshcd_hba_stop() (HCE = 0) before calling ufshcd_complete_requests(hba, true) -> ufshcd_mcq_compl_pending_transfer(hba, true) -> ufshcd_mcq_force_compl_one() -> ufshcd_mcq_compl_all_cqes_lock(). Because ufshcd_mcq_force_compl_one() is its sole caller, ufshcd_mcq_compl_all_cqes_lock() always runs with HCE = 0. Despite the comment above ufshcd_mcq_compl_all_cqes_lock() stating that reading CQTPy may not be safe with the controller disabled, the function still calls ufshcd_mcq_update_cq_tail_slot() at the end of its sweep: 1. Unsafe CQTPy MMIO read: Calling ufshcd_mcq_update_cq_tail_slot() at the end of the sweep reads CQTPy over MMIO while HCE = 0, directly contradicting the function's documented contract (commit 1373df88d535 ("scsi: ufs: core: Add a comment block above ufshcd_mcq_compl_all_cqes_lock()")) that reading CQTPy may not be safe with the controller disabled. 2. Spurious error logs on empty slots: Sweeping all max_entries slots visits empty entries where command_desc_base_addr is 0, causing ufshcd_mcq_process_cqe() to log unguarded dev_err(hba->dev, "Abnormal CQ entry!\n") messages. Fix both issues in ufshcd_mcq_compl_all_cqes_lock(): - Remove the ufshcd_mcq_update_cq_tail_slot() call and the redundant hwq->cq_head_slot = hwq->cq_tail_slot assignment without replacement. The two indices are already equal after the sweep: they are equal when the sweep starts, since ufshcd_mcq_poll_cqe_lock() consumes entries until cq_head_slot reaches cq_tail_slot, and the sweep advances cq_head_slot by exactly one full ring. Both indices are also reinitialized before the queue is reused. - Extract ufshcd_mcq_compl_cqe() and invoke it only on non-empty slots during full-ring sweeps, keeping "Abnormal CQ entry!" logging strictly for unexpected empty entries in ufshcd_mcq_poll_cqe_lock(). Fixes: ab248643d3d6 ("scsi: ufs: core: Add error handling for MCQ mode") Cc: stable@vger.kernel.org Reviewed-by: Peter Wang Signed-off-by: Stanley Jhu Reviewed-by: Bart Van Assche Link: https://patch.msgid.link/20260928035816.1294326-2-stanleyjhu@google.com Signed-off-by: Martin K. Petersen (Oracle) --- drivers/ufs/core/ufs-mcq.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index 8106d55f4041..df293b66350d 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -312,20 +312,24 @@ static int ufshcd_mcq_get_tag(struct ufs_hba *hba, struct cq_entry *cqe) UFSHCD_NUM_RESERVED; } +static void ufshcd_mcq_compl_cqe(struct ufs_hba *hba, struct cq_entry *cqe) +{ + int tag = ufshcd_mcq_get_tag(hba, cqe); + + ufshcd_compl_one_cqe(hba, tag, cqe); + /* After processing the CQE, mark it as an empty (invalid) entry. */ + cqe->command_desc_base_addr = 0; +} + static void ufshcd_mcq_process_cqe(struct ufs_hba *hba, struct ufs_hw_queue *hwq) { struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq); - if (cqe->command_desc_base_addr) { - int tag = ufshcd_mcq_get_tag(hba, cqe); - - ufshcd_compl_one_cqe(hba, tag, cqe); - /* After processed the cqe, mark it empty (invalid) entry */ - cqe->command_desc_base_addr = 0; - } else { + if (cqe->command_desc_base_addr) + ufshcd_mcq_compl_cqe(hba, cqe); + else dev_err(hba->dev, "Abnormal CQ entry!\n"); - } } /* @@ -333,7 +337,7 @@ static void ufshcd_mcq_process_cqe(struct ufs_hba *hba, * controller disabled (HCE = 0). Reading host controller registers, e.g. the * CQ tail pointer (CQTPy), may not be safe with the host controller disabled. * Hence, iterate over all completion queue entries. This won't result in - * double completions because ufshcd_mcq_process_cqe() clears a CQE after it + * double completions because ufshcd_mcq_compl_cqe() clears a CQE after it * has been processed. */ void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba, @@ -344,13 +348,13 @@ void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba, spin_lock_irqsave(&hwq->cq_lock, flags); while (entries > 0) { - ufshcd_mcq_process_cqe(hba, hwq); + struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq); + + if (cqe->command_desc_base_addr) + ufshcd_mcq_compl_cqe(hba, cqe); ufshcd_mcq_inc_cq_head_slot(hwq); entries--; } - - ufshcd_mcq_update_cq_tail_slot(hwq); - hwq->cq_head_slot = hwq->cq_tail_slot; spin_unlock_irqrestore(&hwq->cq_lock, flags); } From 469fa055664bfd8717b087a80d59c03055789c17 Mon Sep 17 00:00:00 2001 From: Stanley Jhu Date: Mon, 28 Sep 2026 11:58:16 +0800 Subject: [PATCH 6/6] scsi: ufs: core: Decouple CQ sweep from request iterator in MCQ In MCQ mode, ufshcd_mcq_compl_pending_transfer() uses blk_mq_tagset_busy_iter() to iterate over busy requests during error recovery and host reset. However, both iterator callbacks perform whole-queue operations redundantly for each visited request: - force_compl == true: ufshcd_mcq_force_compl_one() calls ufshcd_mcq_compl_all_cqes_lock() on every busy request, sweeping the entire completion ring (hwq->max_entries slots) once per active request under spin_lock_irqsave even though the first sweep already cleared all completion entries. - force_compl == false: ufshcd_mcq_compl_one() acquires cq_lock and polls CQTPy over MMIO via ufshcd_mcq_poll_cqe_lock() for every busy request without doing any per-request work. Sweep or poll each hardware queue (hba->uhq[i]) once at the start of ufshcd_mcq_compl_pending_transfer(). When force_compl is true, run blk_mq_tagset_busy_iter() afterward to complete residual in-flight requests with DID_REQUEUE, and remove the now-unused ufshcd_mcq_compl_one() callback. Fixes: ab248643d3d6 ("scsi: ufs: core: Add error handling for MCQ mode") Cc: stable@vger.kernel.org Reviewed-by: Bart Van Assche Signed-off-by: Stanley Jhu Reviewed-by: Peter Wang Link: https://patch.msgid.link/20260928035816.1294326-3-stanleyjhu@google.com Signed-off-by: Martin K. Petersen (Oracle) --- drivers/ufs/core/ufshcd.c | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 79b6a2d9b522..ca8329052cbc 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -6044,8 +6044,6 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) if (blk_mq_is_reserved_rq(rq) || !hwq) return true; - ufshcd_mcq_compl_all_cqes_lock(hba, hwq); - /* * For those cmds of which the cqes are not present in the cq, complete * them explicitly. @@ -6061,19 +6059,6 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) return true; } -static bool ufshcd_mcq_compl_one(struct request *rq, void *priv) -{ - struct scsi_device *sdev = rq->q->queuedata; - struct Scsi_Host *shost = sdev->host; - struct ufs_hba *hba = shost_priv(shost); - struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); - - if (!blk_mq_is_reserved_rq(rq) && hwq) - ufshcd_mcq_poll_cqe_lock(hba, hwq); - - return true; -} - /** * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is * invoked from the error handler context or ufshcd_host_reset_and_restore() @@ -6088,10 +6073,18 @@ static bool ufshcd_mcq_compl_one(struct request *rq, void *priv) static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, bool force_compl) { - blk_mq_tagset_busy_iter(&hba->host->tag_set, - force_compl ? ufshcd_mcq_force_compl_one : - ufshcd_mcq_compl_one, - NULL); + int i; + + for (i = 0; i < hba->nr_hw_queues; i++) { + if (force_compl) + ufshcd_mcq_compl_all_cqes_lock(hba, &hba->uhq[i]); + else + ufshcd_mcq_poll_cqe_lock(hba, &hba->uhq[i]); + } + + if (force_compl) + blk_mq_tagset_busy_iter(&hba->host->tag_set, + ufshcd_mcq_force_compl_one, NULL); } /**