From 25423cda145f9ed6ee4a72d9f2603ac2a4685e74 Mon Sep 17 00:00:00 2001 From: Bhanu Seshu Kumar Valluri Date: Tue, 14 Oct 2025 08:11:09 +0530 Subject: [PATCH 1/5] PCI: endpoint: pci-epf-test: Fix sleeping function being called from atomic context When Root Complex (RC) triggers a Doorbell interrupt to Endpoint (EP), it triggers the below warning in the EP: BUG: sleeping function called from invalid context at kernel/locking/mutex.c:271 Call trace: __might_resched+0x130/0x158 __might_sleep+0x70/0x88 mutex_lock+0x2c/0x80 pci_epc_get_msi+0x78/0xd8 pci_epf_test_raise_irq.isra.0+0x74/0x138 pci_epf_test_doorbell_handler+0x34/0x50 The BUG arises because the EP's pci_epf_test_doorbell_handler() which is running in the hard IRQ context is making an indirect call to pci_epc_get_msi(), which uses mutex inside. To fix the issue, convert the hard IRQ handler to a threaded IRQ handler to allow it to call functions that can sleep during bottom half execution. Also, register the threaded IRQ handler with IRQF_ONESHOT to keep the interrupt line disabled until the threaded IRQ handler completes execution. Fixes: eff0c286aa91 ("PCI: endpoint: pci-epf-test: Add doorbell test support") Signed-off-by: Bhanu Seshu Kumar Valluri [mani: reworded description a bit] Signed-off-by: Manivannan Sadhasivam Reviewed-by: Niklas Cassel Reviewed-by: Frank Li Link: https://patch.msgid.link/20251014024109.42287-1-bhanuseshukumar@gmail.com --- drivers/pci/endpoint/functions/pci-epf-test.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index 31617772ad51..b05e8db575c3 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -730,8 +730,9 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test, if (bar < BAR_0) goto err_doorbell_cleanup; - ret = request_irq(epf->db_msg[0].virq, pci_epf_test_doorbell_handler, 0, - "pci-ep-test-doorbell", epf_test); + ret = request_threaded_irq(epf->db_msg[0].virq, NULL, + pci_epf_test_doorbell_handler, IRQF_ONESHOT, + "pci-ep-test-doorbell", epf_test); if (ret) { dev_err(&epf->dev, "Failed to request doorbell IRQ: %d\n", From 483768846d66c04354898f00bcdaad58a3763be2 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Wed, 15 Oct 2025 11:27:28 -0400 Subject: [PATCH 2/5] PCI: endpoint: Rename 'epf_bar::aligned_size' to 'epf_bar:mem_size' Rename the member 'epf_bar::aligned_size' to 'epf_bar::mem_size' to better reflect its purpose. 'aligned_size' was misleading, as it actually represents the backing memory size allocated for the BAR rather than the aligned size. Signed-off-by: Frank Li Signed-off-by: Manivannan Sadhasivam Reviewed-by: Niklas Cassel Link: https://patch.msgid.link/20251015-vntb_msi_doorbell-v6-1-9230298b1910@nxp.com --- drivers/pci/endpoint/pci-epf-core.c | 12 ++++++------ include/linux/pci-epf.h | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c index d54e18872aef..214e3f6e6d0d 100644 --- a/drivers/pci/endpoint/pci-epf-core.c +++ b/drivers/pci/endpoint/pci-epf-core.c @@ -236,13 +236,13 @@ void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar, } dev = epc->dev.parent; - dma_free_coherent(dev, epf_bar[bar].aligned_size, addr, + dma_free_coherent(dev, epf_bar[bar].mem_size, addr, epf_bar[bar].phys_addr); epf_bar[bar].phys_addr = 0; epf_bar[bar].addr = NULL; epf_bar[bar].size = 0; - epf_bar[bar].aligned_size = 0; + epf_bar[bar].mem_size = 0; epf_bar[bar].barno = 0; epf_bar[bar].flags = 0; } @@ -265,7 +265,7 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, enum pci_epc_interface_type type) { u64 bar_fixed_size = epc_features->bar[bar].fixed_size; - size_t aligned_size, align = epc_features->align; + size_t mem_size, align = epc_features->align; struct pci_epf_bar *epf_bar; dma_addr_t phys_addr; struct pci_epc *epc; @@ -297,7 +297,7 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, * it might be different if, for example, the fixed size of a BAR * is smaller than align. */ - aligned_size = align ? ALIGN(size, align) : size; + mem_size = align ? ALIGN(size, align) : size; if (type == PRIMARY_INTERFACE) { epc = epf->epc; @@ -308,7 +308,7 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, } dev = epc->dev.parent; - space = dma_alloc_coherent(dev, aligned_size, &phys_addr, GFP_KERNEL); + space = dma_alloc_coherent(dev, mem_size, &phys_addr, GFP_KERNEL); if (!space) { dev_err(dev, "failed to allocate mem space\n"); return NULL; @@ -317,7 +317,7 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, epf_bar[bar].phys_addr = phys_addr; epf_bar[bar].addr = space; epf_bar[bar].size = size; - epf_bar[bar].aligned_size = aligned_size; + epf_bar[bar].mem_size = mem_size; epf_bar[bar].barno = bar; if (upper_32_bits(size) || epc_features->bar[bar].only_64bit) epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64; diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index 2e85504ba2ba..4022dd080e20 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -115,8 +115,8 @@ struct pci_epf_driver { * @phys_addr: physical address that should be mapped to the BAR * @addr: virtual address corresponding to the @phys_addr * @size: the size of the address space present in BAR - * @aligned_size: the size actually allocated to accommodate the iATU alignment - * requirement + * @mem_size: the size actually allocated to accommodate the iATU alignment + * requirement * @barno: BAR number * @flags: flags that are set for the BAR */ @@ -124,7 +124,7 @@ struct pci_epf_bar { dma_addr_t phys_addr; void *addr; size_t size; - size_t aligned_size; + size_t mem_size; enum pci_barno barno; int flags; }; From f71e2b67b51dcb2fd8c6d566230f17a735770bee Mon Sep 17 00:00:00 2001 From: Frank Li Date: Wed, 15 Oct 2025 11:27:29 -0400 Subject: [PATCH 3/5] PCI: endpoint: Add pci_epf_get_required_bar_size() helper Introduce pci_epf_get_required_bar_size() helper to get the required BAR size and backing memory size. This helper will be used to set a fixed MMIO address as the backing memory for a BAR. Since this helper returns both BAR size and the aligned memory size, use two parameters, 'bar_size' and 'aligned_mem_size' to avoid confusion. Signed-off-by: Frank Li [mani: renamed helper to pci_epf_get_required_bar_size(), reworded description] Signed-off-by: Manivannan Sadhasivam Reviewed-by: Niklas Cassel Link: https://patch.msgid.link/20251015-vntb_msi_doorbell-v6-2-9230298b1910@nxp.com --- drivers/pci/endpoint/pci-epf-core.c | 74 ++++++++++++++++++----------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c index 214e3f6e6d0d..ec20aabb7f75 100644 --- a/drivers/pci/endpoint/pci-epf-core.c +++ b/drivers/pci/endpoint/pci-epf-core.c @@ -208,6 +208,48 @@ void pci_epf_remove_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf) } EXPORT_SYMBOL_GPL(pci_epf_remove_vepf); +static int pci_epf_get_required_bar_size(struct pci_epf *epf, size_t *bar_size, + size_t *aligned_mem_size, + enum pci_barno bar, + const struct pci_epc_features *epc_features, + enum pci_epc_interface_type type) +{ + u64 bar_fixed_size = epc_features->bar[bar].fixed_size; + size_t align = epc_features->align; + size_t size = *bar_size; + + if (size < 128) + size = 128; + + /* According to PCIe base spec, min size for a resizable BAR is 1 MB. */ + if (epc_features->bar[bar].type == BAR_RESIZABLE && size < SZ_1M) + size = SZ_1M; + + if (epc_features->bar[bar].type == BAR_FIXED && bar_fixed_size) { + if (size > bar_fixed_size) { + dev_err(&epf->dev, + "requested BAR size is larger than fixed size\n"); + return -ENOMEM; + } + size = bar_fixed_size; + } else { + /* BAR size must be power of two */ + size = roundup_pow_of_two(size); + } + + *bar_size = size; + + /* + * The EPC's BAR start address must meet alignment requirements. In most + * cases, the alignment will match the BAR size. However, differences + * can occur—for example, when the fixed BAR size (e.g., 128 bytes) is + * smaller than the required alignment (e.g., 4 KB). + */ + *aligned_mem_size = align ? ALIGN(size, align) : size; + + return 0; +} + /** * pci_epf_free_space() - free the allocated PCI EPF register space * @epf: the EPF device from whom to free the memory @@ -264,40 +306,16 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, const struct pci_epc_features *epc_features, enum pci_epc_interface_type type) { - u64 bar_fixed_size = epc_features->bar[bar].fixed_size; - size_t mem_size, align = epc_features->align; struct pci_epf_bar *epf_bar; dma_addr_t phys_addr; struct pci_epc *epc; struct device *dev; + size_t mem_size; void *space; - if (size < 128) - size = 128; - - /* According to PCIe base spec, min size for a resizable BAR is 1 MB. */ - if (epc_features->bar[bar].type == BAR_RESIZABLE && size < SZ_1M) - size = SZ_1M; - - if (epc_features->bar[bar].type == BAR_FIXED && bar_fixed_size) { - if (size > bar_fixed_size) { - dev_err(&epf->dev, - "requested BAR size is larger than fixed size\n"); - return NULL; - } - size = bar_fixed_size; - } else { - /* BAR size must be power of two */ - size = roundup_pow_of_two(size); - } - - /* - * Allocate enough memory to accommodate the iATU alignment - * requirement. In most cases, this will be the same as .size but - * it might be different if, for example, the fixed size of a BAR - * is smaller than align. - */ - mem_size = align ? ALIGN(size, align) : size; + if (pci_epf_get_required_bar_size(epf, &size, &mem_size, bar, + epc_features, type)) + return NULL; if (type == PRIMARY_INTERFACE) { epc = epf->epc; From 0bfc6758f213a701bd662982de86f0032b51f18c Mon Sep 17 00:00:00 2001 From: Frank Li Date: Wed, 15 Oct 2025 11:27:30 -0400 Subject: [PATCH 4/5] PCI: endpoint: Add pci_epf_assign_bar_space() API Add pci_epf_assign_bar_space() API to allow setting any MMIO address as the BAR memory space, such as an MSI message base address. This API also conforms to the BAR base address and size alignment restrictions enforced by the PCI spec r6.0, sec 7.5.1.2.1. Signed-off-by: Frank Li [mani: removed unused epc var, reworded kdoc, comments and description] Signed-off-by: Manivannan Sadhasivam Link: https://patch.msgid.link/20251015-vntb_msi_doorbell-v6-3-9230298b1910@nxp.com --- drivers/pci/endpoint/pci-epf-core.c | 77 +++++++++++++++++++++++++++++ include/linux/pci-epf.h | 6 +++ 2 files changed, 83 insertions(+) diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c index ec20aabb7f75..9a505c796370 100644 --- a/drivers/pci/endpoint/pci-epf-core.c +++ b/drivers/pci/endpoint/pci-epf-core.c @@ -346,6 +346,83 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, } EXPORT_SYMBOL_GPL(pci_epf_alloc_space); +/** + * pci_epf_assign_bar_space() - Assign PCI EPF BAR space + * @epf: EPF device to assign the BAR memory + * @size: Size of the memory that has to be assigned + * @bar: BAR number for which the memory is assigned + * @epc_features: Features provided by the EPC specific to this EPF + * @type: Identifies if the assignment is for primary EPC or secondary EPC + * @bar_addr: Address to be assigned for the @bar + * + * Invoke to assign memory for the PCI EPF BAR. + * Flag PCI_BASE_ADDRESS_MEM_TYPE_64 will automatically get set if the BAR + * can only be a 64-bit BAR, or if the requested size is larger than 2 GB. + */ +int pci_epf_assign_bar_space(struct pci_epf *epf, size_t size, + enum pci_barno bar, + const struct pci_epc_features *epc_features, + enum pci_epc_interface_type type, + dma_addr_t bar_addr) +{ + size_t bar_size, aligned_mem_size; + struct pci_epf_bar *epf_bar; + dma_addr_t limit; + int pos; + + if (!size) + return -EINVAL; + + limit = bar_addr + size - 1; + + /* + * Bits: 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 + * bar_addr: U U U U U U 0 X X X X X X X X X + * limit: U U U U U U 1 X X X X X X X X X + * + * bar_addr^limit 0 0 0 0 0 0 1 X X X X X X X X X + * + * U: unchanged address bits in range [bar_addr, limit] + * X: bit 0 or 1 + * + * (bar_addr^limit) & BIT_ULL(pos) will find the first set bit from MSB + * (pos). And value of (2 ^ pos) should be able to cover the BAR range. + */ + for (pos = 8 * sizeof(dma_addr_t) - 1; pos > 0; pos--) + if ((limit ^ bar_addr) & BIT_ULL(pos)) + break; + + if (pos == 8 * sizeof(dma_addr_t) - 1) + return -EINVAL; + + bar_size = BIT_ULL(pos + 1); + if (pci_epf_get_required_bar_size(epf, &bar_size, &aligned_mem_size, + bar, epc_features, type)) + return -ENOMEM; + + if (type == PRIMARY_INTERFACE) + epf_bar = epf->bar; + else + epf_bar = epf->sec_epc_bar; + + epf_bar[bar].phys_addr = ALIGN_DOWN(bar_addr, aligned_mem_size); + + if (epf_bar[bar].phys_addr + bar_size < limit) + return -ENOMEM; + + epf_bar[bar].addr = NULL; + epf_bar[bar].size = bar_size; + epf_bar[bar].mem_size = aligned_mem_size; + epf_bar[bar].barno = bar; + if (upper_32_bits(size) || epc_features->bar[bar].only_64bit) + epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64; + else + epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_32; + + return 0; +} +EXPORT_SYMBOL_GPL(pci_epf_assign_bar_space); + static void pci_epf_remove_cfs(struct pci_epf_driver *driver) { struct config_group *group, *tmp; diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index 4022dd080e20..48f68c4dcfa5 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -242,6 +242,12 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar, enum pci_epc_interface_type type); +int pci_epf_assign_bar_space(struct pci_epf *epf, size_t size, + enum pci_barno bar, + const struct pci_epc_features *epc_features, + enum pci_epc_interface_type type, + dma_addr_t bar_addr); + int pci_epf_align_inbound_addr(struct pci_epf *epf, enum pci_barno bar, u64 addr, dma_addr_t *base, size_t *off); int pci_epf_bind(struct pci_epf *epf); From dc693d60664470ec47188c328055d80e8ce7ea44 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Wed, 15 Oct 2025 11:27:31 -0400 Subject: [PATCH 5/5] PCI: endpoint: pci-epf-vntb: Add MSI doorbell support Add MSI doorbell support to reduce latency between PCI host and EP. Before this change: ping 169.254.172.137 64 bytes from 169.254.172.137: icmp_seq=1 ttl=64 time=0.575 ms 64 bytes from 169.254.172.137: icmp_seq=2 ttl=64 time=1.80 ms 64 bytes from 169.254.172.137: icmp_seq=3 ttl=64 time=8.19 ms 64 bytes from 169.254.172.137: icmp_seq=4 ttl=64 time=2.00 ms After this change: ping 169.254.144.71 64 bytes from 169.254.144.71: icmp_seq=1 ttl=64 time=0.215 ms 64 bytes from 169.254.144.71: icmp_seq=2 ttl=64 time=0.456 ms 64 bytes from 169.254.144.71: icmp_seq=3 ttl=64 time=0.448 ms Also, change the data type of 'epf_ntb::db' to 'atomic_64' as multiple doorbell interrupts might trigger at the same time. Signed-off-by: Frank Li [mani: added 'pci_epf_' prefix to doorbell IRQ name, minor code cleanups] Signed-off-by: Manivannan Sadhasivam Link: https://patch.msgid.link/20251015-vntb_msi_doorbell-v6-4-9230298b1910@nxp.com --- drivers/pci/endpoint/functions/pci-epf-vntb.c | 153 ++++++++++++++++-- 1 file changed, 136 insertions(+), 17 deletions(-) diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c index 83e9ab10f9c4..3ecc5059f92b 100644 --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c @@ -36,11 +36,13 @@ * PCIe Root Port PCI EP */ +#include #include #include #include #include +#include #include #include #include @@ -126,12 +128,13 @@ struct epf_ntb { u32 db_count; u32 spad_count; u64 mws_size[MAX_MW]; - u64 db; + atomic64_t db; u32 vbus_number; u16 vntb_pid; u16 vntb_vid; bool linkup; + bool msi_doorbell; u32 spad_size; enum pci_barno epf_ntb_bar[VNTB_BAR_NUM]; @@ -258,9 +261,9 @@ static void epf_ntb_cmd_handler(struct work_struct *work) ntb = container_of(work, struct epf_ntb, cmd_handler.work); - for (i = 1; i < ntb->db_count; i++) { + for (i = 1; i < ntb->db_count && !ntb->msi_doorbell; i++) { if (ntb->epf_db[i]) { - ntb->db |= 1 << (i - 1); + atomic64_or(1 << (i - 1), &ntb->db); ntb_db_event(&ntb->ntb, i); ntb->epf_db[i] = 0; } @@ -319,7 +322,21 @@ static void epf_ntb_cmd_handler(struct work_struct *work) reset_handler: queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler, - msecs_to_jiffies(5)); + ntb->msi_doorbell ? msecs_to_jiffies(500) : msecs_to_jiffies(5)); +} + +static irqreturn_t epf_ntb_doorbell_handler(int irq, void *data) +{ + struct epf_ntb *ntb = data; + int i; + + for (i = 1; i < ntb->db_count; i++) + if (irq == ntb->epf->db_msg[i].virq) { + atomic64_or(1 << (i - 1), &ntb->db); + ntb_db_event(&ntb->ntb, i); + } + + return IRQ_HANDLED; } /** @@ -500,6 +517,94 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb) return 0; } +static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb, + struct pci_epf_bar *db_bar, + const struct pci_epc_features *epc_features, + enum pci_barno barno) +{ + struct pci_epf *epf = ntb->epf; + dma_addr_t low, high; + struct msi_msg *msg; + size_t sz; + int ret; + int i; + + ret = pci_epf_alloc_doorbell(epf, ntb->db_count); + if (ret) + return ret; + + for (i = 0; i < ntb->db_count; i++) { + ret = request_irq(epf->db_msg[i].virq, epf_ntb_doorbell_handler, + 0, "pci_epf_vntb_db", ntb); + + if (ret) { + dev_err(&epf->dev, + "Failed to request doorbell IRQ: %d\n", + epf->db_msg[i].virq); + goto err_free_irq; + } + } + + msg = &epf->db_msg[0].msg; + + high = 0; + low = (u64)msg->address_hi << 32 | msg->address_lo; + + for (i = 0; i < ntb->db_count; i++) { + struct msi_msg *msg = &epf->db_msg[i].msg; + dma_addr_t addr = (u64)msg->address_hi << 32 | msg->address_lo; + + low = min(low, addr); + high = max(high, addr); + } + + sz = high - low + sizeof(u32); + + ret = pci_epf_assign_bar_space(epf, sz, barno, epc_features, 0, low); + if (ret) { + dev_err(&epf->dev, "Failed to assign Doorbell BAR space\n"); + goto err_free_irq; + } + + ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, + ntb->epf->vfunc_no, db_bar); + if (ret) { + dev_err(&epf->dev, "Failed to set Doorbell BAR\n"); + goto err_free_irq; + } + + for (i = 0; i < ntb->db_count; i++) { + struct msi_msg *msg = &epf->db_msg[i].msg; + dma_addr_t addr; + size_t offset; + + ret = pci_epf_align_inbound_addr(epf, db_bar->barno, + ((u64)msg->address_hi << 32) | msg->address_lo, + &addr, &offset); + + if (ret) { + ntb->msi_doorbell = false; + goto err_free_irq; + } + + ntb->reg->db_data[i] = msg->data; + ntb->reg->db_offset[i] = offset; + } + + ntb->reg->db_entry_size = 0; + + ntb->msi_doorbell = true; + + return 0; + +err_free_irq: + for (i--; i >= 0; i--) + free_irq(epf->db_msg[i].virq, ntb); + + pci_epf_free_doorbell(ntb->epf); + return ret; +} + /** * epf_ntb_db_bar_init() - Configure Doorbell window BARs * @ntb: NTB device that facilitates communication between HOST and VHOST @@ -520,21 +625,25 @@ static int epf_ntb_db_bar_init(struct epf_ntb *ntb) ntb->epf->func_no, ntb->epf->vfunc_no); barno = ntb->epf_ntb_bar[BAR_DB]; - - mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, epc_features, 0); - if (!mw_addr) { - dev_err(dev, "Failed to allocate OB address\n"); - return -ENOMEM; - } - - ntb->epf_db = mw_addr; - epf_bar = &ntb->epf->bar[barno]; - ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar); + ret = epf_ntb_db_bar_init_msi_doorbell(ntb, epf_bar, epc_features, barno); if (ret) { - dev_err(dev, "Doorbell BAR set failed\n"); + /* fall back to polling mode */ + mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, epc_features, 0); + if (!mw_addr) { + dev_err(dev, "Failed to allocate OB address\n"); + return -ENOMEM; + } + + ntb->epf_db = mw_addr; + + ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, + ntb->epf->vfunc_no, epf_bar); + if (ret) { + dev_err(dev, "Doorbell BAR set failed\n"); goto err_alloc_peer_mem; + } } return ret; @@ -554,6 +663,16 @@ static void epf_ntb_db_bar_clear(struct epf_ntb *ntb) { enum pci_barno barno; + if (ntb->msi_doorbell) { + int i; + + for (i = 0; i < ntb->db_count; i++) + free_irq(ntb->epf->db_msg[i].virq, ntb); + } + + if (ntb->epf->db_msg) + pci_epf_free_doorbell(ntb->epf); + barno = ntb->epf_ntb_bar[BAR_DB]; pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0); pci_epc_clear_bar(ntb->epf->epc, @@ -1268,7 +1387,7 @@ static u64 vntb_epf_db_read(struct ntb_dev *ndev) { struct epf_ntb *ntb = ntb_ndev(ndev); - return ntb->db; + return atomic64_read(&ntb->db); } static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx, @@ -1308,7 +1427,7 @@ static int vntb_epf_db_clear(struct ntb_dev *ndev, u64 db_bits) { struct epf_ntb *ntb = ntb_ndev(ndev); - ntb->db &= ~db_bits; + atomic64_and(~db_bits, &ntb->db); return 0; }