diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_csr.h b/drivers/net/ethernet/meta/mpnic/mpnic_csr.h index adf81b5ef49c..44ff664e3f3b 100644 --- a/drivers/net/ethernet/meta/mpnic/mpnic_csr.h +++ b/drivers/net/ethernet/meta/mpnic/mpnic_csr.h @@ -36,11 +36,26 @@ *****************************************************************************/ /* NIC_CORE_TDF */ +#define MPNIC_TWQ_CTL(i, j) (0x0 + 1024 * (i) + 2 * (j)) + /* 0x0 */ +#define MPNIC_TWQ_CTL_RESET CSR_BIT(0) +#define MPNIC_TWQ_CTL_ENABLE CSR_BIT(1) #define MPNIC_TWQ_TAIL(i, j) (0x4 + 1024 * (i) + 2 * (j)) /* 0x10 */ +#define MPNIC_TWQ_SIZE(i, j) (0x10 + 1024 * (i) + 2 * (j)) + /* 0x40 */ +#define MPNIC_TWQ_SIZE_SIZE CSR_GENMASK(3, 0) +#define MPNIC_TWQ_BASE_ADDR(i, j) (0x1c + 1024 * (i) + 2 * (j)) + /* 0x70 */ /* NIC_CORE_TCM */ +#define MPNIC_TCQ_CTL(i) (0x80 + 1024 * (i)) /* 0x200 */ +#define MPNIC_TCQ_CTL_RESET CSR_BIT(0) +#define MPNIC_TCQ_CTL_ENABLE CSR_BIT(1) +#define MPNIC_TCQ_BASE_ADDR(i) (0x86 + 1024 * (i)) /* 0x218 */ #define MPNIC_TCQ_HEAD(i) (0x8e + 1024 * (i)) /* 0x238 */ +#define MPNIC_TCQ_SIZE(i) (0x94 + 1024 * (i)) /* 0x250 */ +#define MPNIC_TCQ_SIZE_SIZE CSR_GENMASK(4, 0) /* NIC_CORE_TIM */ #define MPNIC_TIM_CTL1(i) (0xc0 + 1024 * (i)) /* 0x300 */ @@ -50,6 +65,11 @@ #define MPNIC_TIM_CTL1_MASK CSR_BIT(51) #define MPNIC_TIM_CTL1_MASK_EN CSR_BIT(52) #define MPNIC_TIM_CTL1_TRIGGER CSR_BIT(53) +#define MPNIC_TIM_INTR_MASK(i) (0xc8 + 1024 * (i)) /* 0x320 */ +#define MPNIC_TIM_INTR_MASK_MASK CSR_BIT(0) + +/* NIC_CORE_TIM_PRV */ +#define MPNIC_TIM_CTL(i) (0x100100 + 1024 * (i)) /* 0x400400 */ /* NIC_CORE_RBP_HP_GLBL */ #define MPNIC_BDQ_GLBL_CTL0 0x420080 /* 0x1080200 */ @@ -83,6 +103,8 @@ #define MPNIC_RNI_RCM_CTL 0x427004 /* 0x109c010 */ /* NIC_CORE_TDF_GLBL */ +#define MPNIC_TWQ_IDLE(i) (0x428042 + 2 * (i)) /* 0x10a0108 */ +#define MPNIC_TWQ_IDLE_CNT 32 #define MPNIC_TWQ_DEF_PRI_TWD 0x428082 /* 0x10a0208 */ #define MPNIC_TDF_MEM_INIT_REQ 0x42813a /* 0x10a04e8 */ #define MPNIC_TDF_MEM_INIT_DONE 0x42813c /* 0x10a04f0 */ @@ -100,6 +122,8 @@ #define MPNIC_TQS_SLOWDOWN_CTL_ENABLE CSR_BIT(6) #define MPNIC_TQS_MTU_CTL0 0x42a030 /* 0x10a80c0 */ #define MPNIC_TQS_MTU_CTL1 0x42a032 /* 0x10a80c8 */ +#define MPNIC_TQS_IDLE(i) (0x42a040 + 2 * (i)) /* 0x10a8100 */ +#define MPNIC_TQS_IDLE_CNT 32 #define MPNIC_TQS_SET_P0_MAP0(i) (0x42a082 + 2 * (i)) /* 0x10a8208 */ #define MPNIC_TQS_SET_P0_MAP1(i) (0x42a092 + 2 * (i)) /* 0x10a8248 */ #define MPNIC_TQS_GLBL_SHAPING 0x42a108 /* 0x10a8420 */ @@ -135,10 +159,14 @@ #define MPNIC_TQS_PORT_CTL(i) (0x42a1e4 + 2 * (i)) /* 0x10a8790 */ /* NIC_CORE_TDE_GLBL */ +#define MPNIC_TDE_IDLE(i) (0x42b000 + 2 * (i)) /* 0x10ac000 */ +#define MPNIC_TDE_IDLE_CNT 32 #define MPNIC_TDE_MEM_INIT_REQ 0x42b1ee /* 0x10ac7b8 */ #define MPNIC_TDE_MEM_INIT_DONE 0x42b1f0 /* 0x10ac7c0 */ /* NIC_CORE_TCM_GLBL */ +#define MPNIC_TCQ_IDLE(i) (0x42c09e + 2 * (i)) /* 0x10b0278 */ +#define MPNIC_TCQ_IDLE_CNT 16 #define MPNIC_TCM_MEM_INIT_REQ 0x42c0be /* 0x10b02f8 */ #define MPNIC_TCM_MEM_INIT_DONE 0x42c0c0 /* 0x10b0300 */ diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c index 853976e11b03..aaa3a3e99a13 100644 --- a/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c +++ b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -467,6 +468,180 @@ void mpnic_reset_netif_queues(struct mpnic_net *mpn) } } +static void mpnic_enable_twq(struct mpnic_dev *mpd, struct mpnic_ring *twq) +{ + u32 log_size = fls(twq->size_mask); + u32 i = twq->q_idx; + + /* Reset head/tail */ + mpnic_wr64(mpd, MPNIC_TWQ_CTL(i, 0), MPNIC_TWQ_CTL_RESET); + twq->tail = 0; + twq->head = 0; + twq->deferred_meta = -1; + + /* Store descriptor ring address and size */ + mpnic_wr64(mpd, MPNIC_TWQ_BASE_ADDR(i, 0), twq->dma); + mpnic_wr64(mpd, MPNIC_TWQ_SIZE(i, 0), log_size & MPNIC_TWQ_SIZE_SIZE); + + mpnic_wr64(mpd, MPNIC_TWQ_CTL(i, 0), MPNIC_TWQ_CTL_ENABLE); +} + +static void mpnic_enable_tcq(struct mpnic_dev *mpd, + struct mpnic_napi_vector *nv, + struct mpnic_ring *tcq) +{ + u32 log_size = fls(tcq->size_mask); + u32 i = tcq->q_idx; + + /* Reset head/tail */ + mpnic_wr64(mpd, MPNIC_TCQ_CTL(i), MPNIC_TCQ_CTL_RESET); + tcq->tail = 0; + tcq->head = 0; + + /* Store descriptor ring address and size */ + mpnic_wr64(mpd, MPNIC_TCQ_BASE_ADDR(i), tcq->dma); + mpnic_wr64(mpd, MPNIC_TCQ_SIZE(i), log_size & MPNIC_TCQ_SIZE_SIZE); + + /* Store interrupt information for the completion queue */ + mpnic_wr64(mpd, MPNIC_TIM_CTL(i), nv->v_idx); + mpnic_wr64(mpd, MPNIC_TIM_INTR_MASK(i), 0); + + mpnic_wr64(mpd, MPNIC_TCQ_CTL(i), MPNIC_TCQ_CTL_ENABLE); +} + +void mpnic_enable(struct mpnic_net *mpn) +{ + struct mpnic_dev *mpd = mpn->mpd; + int i, j; + + for (i = 0; i < mpn->num_napi; i++) { + struct mpnic_napi_vector *nv = mpn->napi[i]; + + for (j = 0; j < nv->txt_count; j++) { + mpnic_enable_twq(mpd, &nv->qt[j].sub0); + mpnic_enable_tcq(mpd, nv, &nv->qt[j].cmpl); + } + } + + mpnic_wrfl(mpd); +} + +static void mpnic_disable_twq(struct mpnic_dev *mpd, struct mpnic_ring *txr) +{ + u64 twq_ctl = mpnic_rd64(mpd, MPNIC_TWQ_CTL(txr->q_idx, 0)); + + twq_ctl &= ~MPNIC_TWQ_CTL_ENABLE; + mpnic_wr64(mpd, MPNIC_TWQ_CTL(txr->q_idx, 0), twq_ctl); +} + +static void mpnic_disable_tcq(struct mpnic_dev *mpd, struct mpnic_ring *txr) +{ + mpnic_wr64(mpd, MPNIC_TCQ_CTL(txr->q_idx), 0); + mpnic_wr64(mpd, MPNIC_TIM_INTR_MASK(txr->q_idx), + MPNIC_TIM_INTR_MASK_MASK); +} + +void mpnic_disable(struct mpnic_net *mpn) +{ + struct mpnic_dev *mpd = mpn->mpd; + int i, j; + + for (i = 0; i < mpn->num_napi; i++) { + struct mpnic_napi_vector *nv = mpn->napi[i]; + + for (j = 0; j < nv->txt_count; j++) { + mpnic_disable_twq(mpd, &nv->qt[j].sub0); + mpnic_disable_tcq(mpd, &nv->qt[j].cmpl); + } + } + + mpnic_wrfl(mpd); +} + +struct mpnic_idle_regs { + u32 reg_base; + u8 reg_cnt; + char name[4]; +}; + +static u32 mpnic_non_idle_queues(struct mpnic_dev *mpd, + const struct mpnic_idle_regs *regs, + unsigned int nregs) +{ + u32 non_idle_bitmap = 0; + unsigned int i, j; + + for (i = 0; i < nregs; i++) { + for (j = 0; j < regs[i].reg_cnt; j++) { + if (mpnic_rd64(mpd, regs[i].reg_base + 2 * j) != + ~0ULL) { + non_idle_bitmap |= BIT(i); + break; + } + } + } + + return non_idle_bitmap; +} + +static void mpnic_idle_dump(struct mpnic_dev *mpd, + const struct mpnic_idle_regs *regs, + unsigned int nregs, u32 non_idle_bitmap, int err) +{ + unsigned int i, j; + + dev_err(mpd->dev, "error waiting for queues idle %d\n", err); + for (i = 0; i < nregs; i++) { + if (!(non_idle_bitmap & BIT(i))) + continue; + + dev_err(mpd->dev, "%s block not idle:\n", regs[i].name); + for (j = 0; j < regs[i].reg_cnt; j++) + dev_err(mpd->dev, " 0x%04x: %016llx\n", + regs[i].reg_base + 2 * j, + mpnic_rd64(mpd, regs[i].reg_base + 2 * j)); + } +} + +void mpnic_wait_all_queues_idle(struct mpnic_dev *mpd) +{ + static const struct mpnic_idle_regs queues[] = { + { MPNIC_TWQ_IDLE(0), MPNIC_TWQ_IDLE_CNT, "TWQ" }, + { MPNIC_TQS_IDLE(0), MPNIC_TQS_IDLE_CNT, "TQS" }, + { MPNIC_TDE_IDLE(0), MPNIC_TDE_IDLE_CNT, "TDE" }, + { MPNIC_TCQ_IDLE(0), MPNIC_TCQ_IDLE_CNT, "TCQ" }, + }; + u32 non_idle_bitmap; + int err; + + err = read_poll_timeout(mpnic_non_idle_queues, non_idle_bitmap, + !non_idle_bitmap, 20, 500000, false, mpd, + queues, ARRAY_SIZE(queues)); + if (err) + mpnic_idle_dump(mpd, queues, ARRAY_SIZE(queues), + non_idle_bitmap, err); +} + +void mpnic_flush(struct mpnic_net *mpn) +{ + int i, j; + + for (i = 0; i < mpn->num_napi; i++) { + struct mpnic_napi_vector *nv = mpn->napi[i]; + + for (j = 0; j < nv->txt_count; j++) { + struct mpnic_q_triad *qt = &nv->qt[j]; + struct netdev_queue *txq; + + /* Clean the work queue of unprocessed work */ + mpnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail); + + txq = netdev_get_tx_queue(mpn->netdev, qt->sub0.q_idx); + netdev_tx_reset_queue(txq); + } + } +} + void mpnic_napi_disable(struct mpnic_net *mpn) { int i; diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h index 0d4667666b5b..68a773133fa2 100644 --- a/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h +++ b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h @@ -76,5 +76,9 @@ int mpnic_set_netif_queues(struct mpnic_net *mpn); void mpnic_reset_netif_queues(struct mpnic_net *mpn); void mpnic_napi_enable(struct mpnic_net *mpn); void mpnic_napi_disable(struct mpnic_net *mpn); +void mpnic_enable(struct mpnic_net *mpn); +void mpnic_disable(struct mpnic_net *mpn); +void mpnic_wait_all_queues_idle(struct mpnic_dev *mpd); +void mpnic_flush(struct mpnic_net *mpn); #endif /* _MPNIC_TXRX_H_ */