diff --git a/drivers/net/ethernet/meta/mpnic/Makefile b/drivers/net/ethernet/meta/mpnic/Makefile index 3aa7c6a5ab5e..67d617f49a8a 100644 --- a/drivers/net/ethernet/meta/mpnic/Makefile +++ b/drivers/net/ethernet/meta/mpnic/Makefile @@ -11,4 +11,5 @@ mpnic-y := \ mpnic_init.o \ mpnic_irq.o \ mpnic_pci.o \ + mpnic_txrx.o \ # End of mpnic-y diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_csr.h b/drivers/net/ethernet/meta/mpnic/mpnic_csr.h index 3ddd6acea085..adf81b5ef49c 100644 --- a/drivers/net/ethernet/meta/mpnic/mpnic_csr.h +++ b/drivers/net/ethernet/meta/mpnic/mpnic_csr.h @@ -15,6 +15,13 @@ /* Transmit Work Descriptor Format */ #define MPNIC_TWD_FLAG_REQ_COMPLETION DESC_BIT(37) +#define MPNIC_TWD_ADDR DESC_GENMASK(45, 0) +#define MPNIC_TWD_LEN DESC_GENMASK(63, 48) + +/* Tx Completion Descriptor Format */ +#define MPNIC_TCD_TYPE0_HEAD0 DESC_GENMASK(15, 0) +#define MPNIC_TCD_DONE DESC_BIT(63) + /* Common fields for all DESC_CFG CSRs */ #define MPNIC_DESC_CFG_NUM_DESCS CSR_GENMASK(2, 0) #define MPNIC_DESC_CFG_START_ADDR CSR_GENMASK(19, 8) @@ -28,6 +35,22 @@ * Name Index Address *****************************************************************************/ +/* NIC_CORE_TDF */ +#define MPNIC_TWQ_TAIL(i, j) (0x4 + 1024 * (i) + 2 * (j)) + /* 0x10 */ + +/* NIC_CORE_TCM */ +#define MPNIC_TCQ_HEAD(i) (0x8e + 1024 * (i)) /* 0x238 */ + +/* NIC_CORE_TIM */ +#define MPNIC_TIM_CTL1(i) (0xc0 + 1024 * (i)) /* 0x300 */ +#define MPNIC_TIM_CTL1_UPD_IGN_LONG_EVENT_CNT CSR_BIT(48) +#define MPNIC_TIM_CTL1_UPD_IGN_LONG_TIME_CNT CSR_BIT(49) +#define MPNIC_TIM_CTL1_UPD_IGN_SHORT_TIME_CNT CSR_BIT(50) +#define MPNIC_TIM_CTL1_MASK CSR_BIT(51) +#define MPNIC_TIM_CTL1_MASK_EN CSR_BIT(52) +#define MPNIC_TIM_CTL1_TRIGGER CSR_BIT(53) + /* NIC_CORE_RBP_HP_GLBL */ #define MPNIC_BDQ_GLBL_CTL0 0x420080 /* 0x1080200 */ #define MPNIC_BDQ_GLBL_CTL0_MAX_REQ_SIZE CSR_GENMASK(26, 18) diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_netdev.h b/drivers/net/ethernet/meta/mpnic/mpnic_netdev.h new file mode 100644 index 000000000000..f98adf209b45 --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/mpnic_netdev.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _MPNIC_NETDEV_H_ +#define _MPNIC_NETDEV_H_ + +#include + +#include "mpnic.h" +#include "mpnic_txrx.h" + +struct mpnic_net { + struct mpnic_ring *tx[MPNIC_MAX_TXQS]; + + struct mpnic_napi_vector *napi[MPNIC_MAX_NAPI_VECTORS]; + + struct net_device *netdev; + struct mpnic_dev *mpd; + + u32 txq_size; + + u16 num_napi; + u16 num_tx_queues; +}; + +#endif /* _MPNIC_NETDEV_H_ */ diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c new file mode 100644 index 000000000000..853976e11b03 --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c @@ -0,0 +1,495 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include + +#include "mpnic.h" +#include "mpnic_netdev.h" +#include "mpnic_txrx.h" + +struct mpnic_xmit_cb { + u32 bytecount; + u8 desc_count; +}; + +#define MPNIC_XMIT_CB(__skb) ((struct mpnic_xmit_cb *)((__skb)->cb)) + +/* Leave the interrupt moderation counters alone when arming or masking */ +#define MPNIC_TIM_PARAM_CFG_PRESERVE_MASK \ + (MPNIC_TIM_CTL1_UPD_IGN_LONG_EVENT_CNT | \ + MPNIC_TIM_CTL1_UPD_IGN_LONG_TIME_CNT | \ + MPNIC_TIM_CTL1_UPD_IGN_SHORT_TIME_CNT) + +static void mpnic_nv_irq_disable(struct mpnic_napi_vector *nv) +{ + mpnic_wr64(nv->mpd, MPNIC_TIM_CTL1(nv->qt[0].cmpl.q_idx), + MPNIC_TIM_PARAM_CFG_PRESERVE_MASK | + MPNIC_TIM_CTL1_MASK_EN | MPNIC_TIM_CTL1_MASK); +} + +static void mpnic_nv_irq_rearm(struct mpnic_napi_vector *nv) +{ + /* Rearming a single queue on a given IRQ rearms all the other + * queues mapped to the same IRQ. + */ + mpnic_wr64(nv->mpd, MPNIC_TIM_CTL1(nv->qt[0].cmpl.q_idx), + MPNIC_TIM_PARAM_CFG_PRESERVE_MASK | MPNIC_TIM_CTL1_MASK_EN); +} + +static void mpnic_nv_irq_trigger(struct mpnic_napi_vector *nv) +{ + mpnic_wr64(nv->mpd, MPNIC_TIM_CTL1(nv->qt[0].cmpl.q_idx), + MPNIC_TIM_PARAM_CFG_PRESERVE_MASK | + MPNIC_TIM_CTL1_MASK_EN | MPNIC_TIM_CTL1_TRIGGER); +} + +static unsigned int mpnic_desc_unused(struct mpnic_ring *ring) +{ + return (ring->head - ring->tail - 1) & ring->size_mask; +} + +static struct netdev_queue *mpnic_txring_txq(const struct net_device *dev, + const struct mpnic_ring *ring) +{ + return netdev_get_tx_queue(dev, ring->q_idx); +} + +static void mpnic_unmap_single_twd(struct device *dev, __le64 *twd) +{ + u64 raw_twd = le64_to_cpu(*twd); + + dma_unmap_single(dev, FIELD_GET(MPNIC_TWD_ADDR, raw_twd), + FIELD_GET(MPNIC_TWD_LEN, raw_twd), DMA_TO_DEVICE); +} + +static void mpnic_unmap_page_twd(struct device *dev, __le64 *twd) +{ + u64 raw_twd = le64_to_cpu(*twd); + + dma_unmap_page(dev, FIELD_GET(MPNIC_TWD_ADDR, raw_twd), + FIELD_GET(MPNIC_TWD_LEN, raw_twd), DMA_TO_DEVICE); +} + +static void mpnic_clean_twq0(struct mpnic_napi_vector *nv, int napi_budget, + struct mpnic_ring *ring, bool discard, + unsigned int hw_head) +{ + u64 total_bytes = 0, total_packets = 0; + unsigned int head = ring->head; + struct netdev_queue *txq; + unsigned int clean_desc; + + clean_desc = (hw_head - head) & ring->size_mask; + + while (clean_desc) { + struct sk_buff *skb = ring->tx_buf[head]; + unsigned int desc_cnt; + + desc_cnt = MPNIC_XMIT_CB(skb)->desc_count; + if (desc_cnt > clean_desc) + break; + + ring->tx_buf[head] = NULL; + + clean_desc -= desc_cnt; + + /* Step over the metadata descriptor */ + head++; + head &= ring->size_mask; + desc_cnt--; + + mpnic_unmap_single_twd(nv->dev, &ring->desc[head]); + head++; + head &= ring->size_mask; + desc_cnt--; + + while (desc_cnt--) { + mpnic_unmap_page_twd(nv->dev, &ring->desc[head]); + head++; + head &= ring->size_mask; + } + + total_bytes += MPNIC_XMIT_CB(skb)->bytecount; + total_packets++; + + napi_consume_skb(skb, napi_budget); + } + + if (!total_bytes) + return; + + ring->head = head; + + if (discard) + return; + + txq = mpnic_txring_txq(nv->napi.dev, ring); + netif_txq_completed_wake(txq, total_packets, total_bytes, + mpnic_desc_unused(ring), + MPNIC_TX_DESC_WAKEUP); +} + +static void mpnic_commit_cq_head(struct mpnic_ring *cmpl) +{ + u32 head = cmpl->head; + + /* The tail shadows the last value written to the doorbell, so a + * completion queue which has not moved costs no MMIO write. + */ + if (cmpl->tail != head) { + cmpl->tail = head; + writeq(head & cmpl->size_mask, cmpl->doorbell); + } +} + +static void mpnic_clean_tcq(struct mpnic_napi_vector *nv, + struct mpnic_q_triad *qt, int napi_budget) +{ + struct mpnic_ring *cmpl = &qt->cmpl; + __le64 *raw_tcd, done; + u32 head = cmpl->head; + s32 head0 = -1; + + done = (head & (cmpl->size_mask + 1)) ? 0 : cpu_to_le64(MPNIC_TCD_DONE); + raw_tcd = &cmpl->desc[head & cmpl->size_mask]; + + /* Walk the completion queue collecting the heads reported by NIC. + * Only the first work queue is enabled and no packet asks for a + * timestamp, so every completion is a plain head update and the + * descriptor type does not have to be decoded. + */ + while ((*raw_tcd & cpu_to_le64(MPNIC_TCD_DONE)) == done) { + u64 tcd; + + dma_rmb(); + + tcd = le64_to_cpu(*raw_tcd); + head0 = FIELD_GET(MPNIC_TCD_TYPE0_HEAD0, tcd); + + raw_tcd++; + head++; + + if (unlikely(!(head & cmpl->size_mask))) { + done ^= cpu_to_le64(MPNIC_TCD_DONE); + raw_tcd = &cmpl->desc[0]; + } + } + + cmpl->head = head; + + if (head0 >= 0) + mpnic_clean_twq0(nv, napi_budget, &qt->sub0, false, head0); +} + +static int mpnic_poll(struct napi_struct *napi, int budget) +{ + struct mpnic_napi_vector *nv = container_of(napi, + struct mpnic_napi_vector, + napi); + int i; + + for (i = 0; i < nv->txt_count; i++) + mpnic_clean_tcq(nv, &nv->qt[i], budget); + + for (i = 0; i < nv->txt_count; i++) + mpnic_commit_cq_head(&nv->qt[i].cmpl); + + if (likely(napi_complete_done(napi, 0))) + mpnic_nv_irq_rearm(nv); + + return 0; +} + +static irqreturn_t mpnic_msix_clean_rings(int __always_unused irq, void *data) +{ + struct mpnic_napi_vector *nv = data; + + napi_schedule_irqoff(&nv->napi); + + return IRQ_HANDLED; +} + +static void mpnic_free_napi_vector(struct mpnic_net *mpn, + struct mpnic_napi_vector *nv) +{ + int i; + + for (i = 0; i < nv->txt_count; i++) + mpn->tx[nv->qt[i].sub0.q_idx] = NULL; + + mpnic_free_irq(nv->mpd, nv->v_idx, nv); + netif_napi_del_locked(&nv->napi); + mpn->napi[nv->v_idx - MPNIC_NON_NAPI_VECTORS] = NULL; + kfree(nv); +} + +void mpnic_free_napi_vectors(struct mpnic_net *mpn) +{ + int i; + + for (i = 0; i < mpn->num_napi; i++) + if (mpn->napi[i]) + mpnic_free_napi_vector(mpn, mpn->napi[i]); +} + +static void mpnic_ring_init(struct mpnic_ring *ring, u32 __iomem *doorbell, + int q_idx) +{ + ring->doorbell = doorbell; + ring->q_idx = q_idx; +} + +static int mpnic_alloc_napi_vector(struct mpnic_dev *mpd, + struct mpnic_net *mpn, unsigned int idx) +{ + u32 __iomem *uc_addr = READ_ONCE(mpd->uc_addr0); + struct mpnic_napi_vector *nv; + int err; + + /* Doorbells are plain pointers into the register window, they have + * no way of noticing that it went away. + */ + if (!uc_addr) + return -EIO; + + nv = kzalloc_flex(*nv, qt, 1); + if (!nv) + return -ENOMEM; + + nv->txt_count = 1; + nv->mpd = mpd; + nv->dev = mpd->dev; + nv->v_idx = idx + MPNIC_NON_NAPI_VECTORS; + + mpn->napi[idx] = nv; + netif_napi_add_config_locked(mpn->netdev, &nv->napi, mpnic_poll, idx); + netif_napi_set_irq_locked(&nv->napi, + pci_irq_vector(to_pci_dev(mpd->dev), + nv->v_idx)); + + snprintf(nv->name, sizeof(nv->name), "%s-TxRx-%u", + mpn->netdev->name, idx); + + err = mpnic_request_irq(mpd, nv->v_idx, mpnic_msix_clean_rings, 0, + nv->name, nv); + if (err) + goto err_napi_del; + + mpnic_ring_init(&nv->qt[0].sub0, &uc_addr[MPNIC_TWQ_TAIL(idx, 0)], idx); + mpnic_ring_init(&nv->qt[0].cmpl, &uc_addr[MPNIC_TCQ_HEAD(idx)], idx); + mpn->tx[idx] = &nv->qt[0].sub0; + + return 0; + +err_napi_del: + netif_napi_del_locked(&nv->napi); + mpn->napi[idx] = NULL; + kfree(nv); + return err; +} + +int mpnic_alloc_napi_vectors(struct mpnic_net *mpn) +{ + unsigned int i; + int err; + + for (i = 0; i < mpn->num_napi; i++) { + err = mpnic_alloc_napi_vector(mpn->mpd, mpn, i); + if (err) + goto err_free_vectors; + } + + return 0; + +err_free_vectors: + mpnic_free_napi_vectors(mpn); + + return err; +} + +static void mpnic_free_ring_resources(struct device *dev, + struct mpnic_ring *ring) +{ + kvfree(ring->tx_buf); + ring->tx_buf = NULL; + + /* If size is not set there are no descriptors present */ + if (!ring->size) + return; + + dma_free_coherent(dev, ring->size, ring->desc, ring->dma); + ring->size_mask = 0; + ring->size = 0; +} + +static int mpnic_alloc_ring_desc(struct mpnic_net *mpn, + struct mpnic_ring *ring, u32 count) +{ + struct device *dev = mpn->netdev->dev.parent; + size_t size; + + size = ALIGN(array_size(sizeof(*ring->desc), count), 4096); + + ring->desc = dma_alloc_coherent(dev, size, &ring->dma, + GFP_KERNEL | __GFP_NOWARN); + if (!ring->desc) + return -ENOMEM; + + ring->size_mask = count - 1; + ring->size = size; + + return 0; +} + +static void mpnic_free_tx_qt_resources(struct mpnic_net *mpn, + struct mpnic_q_triad *qt) +{ + struct device *dev = mpn->netdev->dev.parent; + + mpnic_free_ring_resources(dev, &qt->cmpl); + mpnic_free_ring_resources(dev, &qt->sub0); +} + +static int mpnic_alloc_tx_qt_resources(struct mpnic_net *mpn, + struct mpnic_q_triad *qt) +{ + int err; + + err = mpnic_alloc_ring_desc(mpn, &qt->sub0, mpn->txq_size); + if (err) + return err; + + qt->sub0.tx_buf = kvzalloc_objs(*qt->sub0.tx_buf, mpn->txq_size, + GFP_KERNEL | __GFP_NOWARN); + if (!qt->sub0.tx_buf) { + err = -ENOMEM; + goto err_free_qt; + } + + err = mpnic_alloc_ring_desc(mpn, &qt->cmpl, mpn->txq_size); + if (err) + goto err_free_qt; + + return 0; + +err_free_qt: + mpnic_free_tx_qt_resources(mpn, qt); + return err; +} + +static void mpnic_free_nv_resources(struct mpnic_net *mpn, + struct mpnic_napi_vector *nv) +{ + int i; + + for (i = 0; i < nv->txt_count; i++) + mpnic_free_tx_qt_resources(mpn, &nv->qt[i]); +} + +static int mpnic_alloc_nv_resources(struct mpnic_net *mpn, + struct mpnic_napi_vector *nv) +{ + int i, err; + + for (i = 0; i < nv->txt_count; i++) { + err = mpnic_alloc_tx_qt_resources(mpn, &nv->qt[i]); + if (err) + goto err_free_qt_resources; + } + + return 0; + +err_free_qt_resources: + while (i--) + mpnic_free_tx_qt_resources(mpn, &nv->qt[i]); + return err; +} + +void mpnic_free_resources(struct mpnic_net *mpn) +{ + int i; + + for (i = 0; i < mpn->num_napi; i++) + mpnic_free_nv_resources(mpn, mpn->napi[i]); +} + +int mpnic_alloc_resources(struct mpnic_net *mpn) +{ + int i, err; + + for (i = 0; i < mpn->num_napi; i++) { + err = mpnic_alloc_nv_resources(mpn, mpn->napi[i]); + if (err) + goto err_free_resources; + } + + return 0; + +err_free_resources: + while (i--) + mpnic_free_nv_resources(mpn, mpn->napi[i]); + + return err; +} + +int mpnic_set_netif_queues(struct mpnic_net *mpn) +{ + int i, j, err; + + err = netif_set_real_num_tx_queues(mpn->netdev, mpn->num_tx_queues); + if (err) + return err; + + for (i = 0; i < mpn->num_napi; i++) { + struct mpnic_napi_vector *nv = mpn->napi[i]; + + for (j = 0; j < nv->txt_count; j++) + netif_queue_set_napi(mpn->netdev, nv->qt[j].sub0.q_idx, + NETDEV_QUEUE_TYPE_TX, &nv->napi); + } + + return 0; +} + +void mpnic_reset_netif_queues(struct mpnic_net *mpn) +{ + int i, j; + + for (i = 0; i < mpn->num_napi; i++) { + struct mpnic_napi_vector *nv = mpn->napi[i]; + + for (j = 0; j < nv->txt_count; j++) + netif_queue_set_napi(mpn->netdev, nv->qt[j].sub0.q_idx, + NETDEV_QUEUE_TYPE_TX, NULL); + } +} + +void mpnic_napi_disable(struct mpnic_net *mpn) +{ + int i; + + for (i = 0; i < mpn->num_napi; i++) { + napi_disable_locked(&mpn->napi[i]->napi); + + mpnic_nv_irq_disable(mpn->napi[i]); + } +} + +void mpnic_napi_enable(struct mpnic_net *mpn) +{ + int i; + + for (i = 0; i < mpn->num_napi; i++) + napi_enable_locked(&mpn->napi[i]->napi); + + /* Force the first interrupt on each vector to guarantee that any + * completions posted during bringup are processed. + */ + for (i = 0; i < mpn->num_napi; i++) + mpnic_nv_irq_trigger(mpn->napi[i]); + + mpnic_wrfl(mpn->mpd); +} diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h new file mode 100644 index 000000000000..0d4667666b5b --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _MPNIC_TXRX_H_ +#define _MPNIC_TXRX_H_ + +#include +#include +#include +#include +#include + +#include "mpnic.h" + +struct mpnic_net; + +/* Space we have to have available in a work queue to take a packet: + * 1 descriptor per page + * + 1 descriptor for the skb head + * + 1 descriptor for the metadata + * + 7 descriptors to keep the tail out of the head's cacheline + * If we cannot guarantee that we return NETDEV_TX_BUSY. + */ +#define MPNIC_MAX_SKB_DESC (MAX_SKB_FRAGS + 9) +#define MPNIC_TX_DESC_WAKEUP (MPNIC_MAX_SKB_DESC * 2) + +#define MPNIC_MAX_NAPI_VECTORS 1024u + +struct mpnic_ring { + void **tx_buf; /* Packets outstanding in a TWQ */ + + u32 __iomem *doorbell; /* Pointer to CSR space for ring */ + __le64 *desc; /* Descriptor ring memory */ + u16 size_mask; /* Size of ring in descriptors - 1 */ + u16 q_idx; /* Hardware queue index */ + + u32 head, tail; /* Head/Tail of ring */ + + /* TWQ only, index of the metadata descriptor of the last packet + * placed in the ring without ringing the doorbell, -1 if the + * doorbell is in sync with the tail. + */ + s32 deferred_meta; + + /* Slow path fields follow */ + dma_addr_t dma; /* Phys addr of descriptor memory */ + size_t size; /* Size of descriptor ring in memory */ +}; + +/* The device pairs two work queues with one completion queue. On the Tx + * side only the first work queue is used for now, the second one becomes + * the XDP ring. + */ +struct mpnic_q_triad { + struct mpnic_ring sub0, sub1, cmpl; +}; + +struct mpnic_napi_vector { + struct napi_struct napi; + struct device *dev; /* Device for DMA unmapping */ + struct mpnic_dev *mpd; + + u16 v_idx; + u16 txt_count; + + char name[IFNAMSIZ + 11]; + + struct mpnic_q_triad qt[]; +}; + +int mpnic_alloc_napi_vectors(struct mpnic_net *mpn); +void mpnic_free_napi_vectors(struct mpnic_net *mpn); +int mpnic_alloc_resources(struct mpnic_net *mpn); +void mpnic_free_resources(struct mpnic_net *mpn); +int mpnic_set_netif_queues(struct mpnic_net *mpn); +void mpnic_reset_netif_queues(struct mpnic_net *mpn); +void mpnic_napi_enable(struct mpnic_net *mpn); +void mpnic_napi_disable(struct mpnic_net *mpn); + +#endif /* _MPNIC_TXRX_H_ */