diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h index baba3471bf5a..cbb25d11638c 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h @@ -109,34 +109,19 @@ enum { /* Rx Buffer Descriptor Format * - * The layout of this can vary depending on the page size of the system. + * Buffer descriptors describe 4 KiB device pages. A posted page larger than + * 4 KiB is represented by consecutive device-page descriptors. * - * If the page size is 4K then the layout will simply consist of ID for - * the 16 most significant bits, and the lower 46 are essentially the page - * address with the lowest 12 bits being reserved 0 due to the fact that - * a page will be aligned. - * - * If the page size is larger than 4K then the lower n bits of the ID and - * page address will be reserved for the fragment ID. This fragment will - * be 4K in size and will be used to index both the DMA address and the ID - * by the same amount. + * The address field stores the 4 KiB-aligned DMA address. The ID field stores + * the software page ID, with the low n bits used as the device-page ID when a + * posted page spans multiple device pages. The driver increments both the + * address and ID by one device page for each descriptor belonging to a posted + * page. */ #define FBNIC_BD_DESC_ADDR_MASK DESC_GENMASK(45, 12) #define FBNIC_BD_DESC_ID_MASK DESC_GENMASK(63, 48) -#define FBNIC_BD_FRAG_SIZE \ +#define FBNIC_BD_PAGE_SIZE \ (FBNIC_BD_DESC_ADDR_MASK & ~(FBNIC_BD_DESC_ADDR_MASK - 1)) -#define FBNIC_BD_FRAG_COUNT \ - (PAGE_SIZE / FBNIC_BD_FRAG_SIZE) -#define FBNIC_BD_FRAG_ADDR_MASK \ - (FBNIC_BD_DESC_ADDR_MASK & \ - ~(FBNIC_BD_DESC_ADDR_MASK * FBNIC_BD_FRAG_COUNT)) -#define FBNIC_BD_FRAG_ID_MASK \ - (FBNIC_BD_DESC_ID_MASK & \ - ~(FBNIC_BD_DESC_ID_MASK * FBNIC_BD_FRAG_COUNT)) -#define FBNIC_BD_PAGE_ADDR_MASK \ - (FBNIC_BD_DESC_ADDR_MASK & ~FBNIC_BD_FRAG_ADDR_MASK) -#define FBNIC_BD_PAGE_ID_MASK \ - (FBNIC_BD_DESC_ID_MASK & ~FBNIC_BD_FRAG_ID_MASK) /* Rx Completion Queue Descriptors */ #define FBNIC_RCD_TYPE_MASK DESC_GENMASK(62, 61) @@ -151,9 +136,6 @@ enum { /* Address/Length Completion Descriptors */ #define FBNIC_RCD_AL_BUFF_ID_MASK DESC_GENMASK(15, 0) -#define FBNIC_RCD_AL_BUFF_FRAG_MASK (FBNIC_BD_FRAG_COUNT - 1) -#define FBNIC_RCD_AL_BUFF_PAGE_MASK \ - (FBNIC_RCD_AL_BUFF_ID_MASK & ~FBNIC_RCD_AL_BUFF_FRAG_MASK) #define FBNIC_RCD_AL_BUFF_LEN_MASK DESC_GENMASK(28, 16) #define FBNIC_RCD_AL_BUFF_OFF_MASK DESC_GENMASK(43, 32) #define FBNIC_RCD_AL_PAGE_FIN DESC_BIT(60) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c index 6edfa0aa69f1..f9e879be254d 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c @@ -181,8 +181,8 @@ static int fbnic_dbg_tcq_desc_seq_show(struct seq_file *s, void *v) static int fbnic_dbg_bdq_desc_seq_show(struct seq_file *s, void *v) { struct fbnic_ring *ring = s->private; + unsigned int i, desc_count; char hdr[80]; - int i; /* Generate header on first entry */ fbnic_dbg_ring_show(s); @@ -197,7 +197,8 @@ static int fbnic_dbg_bdq_desc_seq_show(struct seq_file *s, void *v) return 0; } - for (i = 0; i < (ring->size_mask + 1) * FBNIC_BD_FRAG_COUNT; i++) { + desc_count = (ring->size_mask + 1) * fbnic_bd_page_count(ring); + for (i = 0; i < desc_count; i++) { u64 bd = le64_to_cpu(ring->desc[i]); seq_printf(s, "%04x %#04llx %#014llx\n", i, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c index 10bf99be3f24..8a6703afca38 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c @@ -773,6 +773,10 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) fbn->ppq_size = FBNIC_PPQ_SIZE_DEFAULT; fbn->rcq_size = FBNIC_RCQ_SIZE_DEFAULT; + netdev->cfg->rings.rx_pending = fbn->rcq_size; + netdev->cfg->rings.rx_mini_pending = fbn->hpq_size; + netdev->cfg->rings.rx_jumbo_pending = fbn->ppq_size; + fbn->tx_usecs = FBNIC_TX_USECS_DEFAULT; fbn->rx_usecs = FBNIC_RX_USECS_DEFAULT; fbn->rx_max_frames = FBNIC_RX_FRAMES_DEFAULT; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index b41ff7c88a0a..c62d885a678a 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -896,19 +896,31 @@ static void fbnic_clean_bdq(struct fbnic_ring *ring, unsigned int hw_head, ring->head = head; } +static u16 fbnic_rcd_bd_idx(const struct fbnic_ring *bdq, u64 rcd) +{ + return FIELD_GET(FBNIC_RCD_AL_BUFF_ID_MASK, rcd) >> bdq->bd_page_shift; +} + +static unsigned int fbnic_rcd_bd_page_offset(const struct fbnic_ring *bdq, + u64 rcd) +{ + u16 id = FIELD_GET(FBNIC_RCD_AL_BUFF_ID_MASK, rcd); + u16 page_id = id & (fbnic_bd_page_count(bdq) - 1); + + return page_id * FBNIC_BD_PAGE_SIZE; +} + static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, netmem_ref netmem) { - __le64 *bdq_desc = &bdq->desc[id * FBNIC_BD_FRAG_COUNT]; + __le64 *bdq_desc = &bdq->desc[id * fbnic_bd_page_count(bdq)]; dma_addr_t dma = page_pool_get_dma_addr_netmem(netmem); - u64 bd, i = FBNIC_BD_FRAG_COUNT; + u64 bd, i = fbnic_bd_page_count(bdq); - bd = (FBNIC_BD_PAGE_ADDR_MASK & dma) | - FIELD_PREP(FBNIC_BD_PAGE_ID_MASK, id); + bd = (FBNIC_BD_DESC_ADDR_MASK & dma) | + FIELD_PREP(FBNIC_BD_DESC_ID_MASK, (u64)id << bdq->bd_page_shift); - /* In the case that a page size is larger than 4K we will map a - * single page to multiple fragments. The fragments will be - * FBNIC_BD_FRAG_COUNT in size and the lower n bits will be use - * to indicate the individual fragment IDs. + /* Posted pages larger than 4 KiB use consecutive device-page IDs in + * the low bits of the software page ID. */ do { *bdq_desc = cpu_to_le64(bd); @@ -953,7 +965,7 @@ static void fbnic_fill_bdq(struct fbnic_ring *bdq) /* Force DMA writes to flush before writing to tail */ dma_wmb(); - writel(i * FBNIC_BD_FRAG_COUNT, bdq->doorbell); + writel(i * fbnic_bd_page_count(bdq), bdq->doorbell); } } @@ -984,26 +996,27 @@ static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd, struct fbnic_pkt_buff *pkt, struct fbnic_q_triad *qt) { - unsigned int hdr_pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); unsigned int hdr_pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); - struct page *page = fbnic_page_pool_get_head(qt, hdr_pg_idx); unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); + unsigned int hdr_pg_idx = fbnic_rcd_bd_idx(&qt->sub0, rcd); unsigned int frame_sz, hdr_pg_start, hdr_pg_end, headroom; unsigned char *hdr_start; + struct page *page; /* data_hard_start should always be NULL when this is called */ WARN_ON_ONCE(pkt->buff.data_hard_start); + page = fbnic_page_pool_get_head(qt, hdr_pg_idx); + /* Short-cut the end calculation if we know page is fully consumed */ hdr_pg_end = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? - FBNIC_BD_FRAG_SIZE : fbnic_hdr_pg_end(hdr_pg_off, len); + FBNIC_BD_PAGE_SIZE : fbnic_hdr_pg_end(hdr_pg_off, len); hdr_pg_start = fbnic_hdr_pg_start(hdr_pg_off); headroom = hdr_pg_off - hdr_pg_start + FBNIC_RX_PAD; frame_sz = hdr_pg_end - hdr_pg_start; xdp_init_buff(&pkt->buff, frame_sz, &qt->xdp_rxq); - hdr_pg_start += (FBNIC_RCD_AL_BUFF_FRAG_MASK & rcd) * - FBNIC_BD_FRAG_SIZE; + hdr_pg_start += fbnic_rcd_bd_page_offset(&qt->sub0, rcd); /* Sync DMA buffer */ dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page), @@ -1024,23 +1037,28 @@ static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd, struct fbnic_pkt_buff *pkt, struct fbnic_q_triad *qt) { - unsigned int pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); unsigned int pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); - netmem_ref netmem = fbnic_page_pool_get_data(qt, pg_idx); + unsigned int pg_idx = fbnic_rcd_bd_idx(&qt->sub1, rcd); unsigned int truesize; + netmem_ref netmem; bool added; - truesize = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? - FBNIC_BD_FRAG_SIZE - pg_off : ALIGN(len, 128); + netmem = fbnic_page_pool_get_data(qt, pg_idx); - pg_off += (FBNIC_RCD_AL_BUFF_FRAG_MASK & rcd) * - FBNIC_BD_FRAG_SIZE; + truesize = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? + FBNIC_BD_PAGE_SIZE - pg_off : + ALIGN(len, FBNIC_RX_PAYLD_ALIGN); + + pg_off += fbnic_rcd_bd_page_offset(&qt->sub1, rcd); /* Sync DMA buffer */ page_pool_dma_sync_netmem_for_cpu(qt->sub1.page_pool, netmem, pg_off, truesize); + /* Consecutive device-page completions from one PPQ page are adjacent + * ranges in the same netmem. + */ added = xdp_buff_add_frag(&pkt->buff, netmem, pg_off, len, truesize); if (unlikely(!added)) { pkt->add_frag_failed = true; @@ -1283,12 +1301,12 @@ static int fbnic_clean_rcq(struct fbnic_napi_vector *nv, switch (FIELD_GET(FBNIC_RCD_TYPE_MASK, rcd)) { case FBNIC_RCD_TYPE_HDR_AL: - head0 = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); + head0 = fbnic_rcd_bd_idx(&qt->sub0, rcd); fbnic_pkt_prepare(nv, rcd, pkt, qt); break; case FBNIC_RCD_TYPE_PAY_AL: - head1 = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); + head1 = fbnic_rcd_bd_idx(&qt->sub1, rcd); fbnic_add_rx_frag(nv, rcd, pkt, qt); break; @@ -1576,7 +1594,7 @@ void fbnic_free_napi_vectors(struct fbnic_net *fbn) static int fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt, - unsigned int rxq_idx) + unsigned int rxq_idx, u32 rx_page_size) { struct page_pool_params pp_params = { .order = 0, @@ -1611,6 +1629,8 @@ fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt, qt->sub0.page_pool = pp; if (netif_rxq_has_unreadable_mp(fbn->netdev, rxq_idx)) { + pp_params.order = get_order(rx_page_size); + pp_params.max_len = rx_page_size; pp_params.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM; pp_params.dma_dir = DMA_FROM_DEVICE; @@ -1629,6 +1649,16 @@ err_destroy_sub0: return PTR_ERR(pp); } +static u8 fbnic_bdq_page_shift(u32 page_size) +{ + return ilog2(page_size / FBNIC_BD_PAGE_SIZE); +} + +static void fbnic_bdq_set_page_size(struct fbnic_ring *bdq, u32 page_size) +{ + bdq->bd_page_shift = fbnic_bdq_page_shift(page_size); +} + static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell, int q_idx, u8 flags) { @@ -1636,6 +1666,7 @@ static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell, ring->doorbell = doorbell; ring->q_idx = q_idx; ring->flags = flags; + fbnic_bdq_set_page_size(ring, PAGE_SIZE); ring->deferred_head = -1; } @@ -1920,12 +1951,12 @@ static int fbnic_alloc_rx_ring_desc(struct fbnic_net *fbn, switch (rxr->doorbell - fbnic_ring_csr_base(rxr)) { case FBNIC_QUEUE_BDQ_HPQ_TAIL: - rxq_size = fbn->hpq_size / FBNIC_BD_FRAG_COUNT; - desc_size *= FBNIC_BD_FRAG_COUNT; + rxq_size = fbn->hpq_size / fbnic_bd_page_count(rxr); + desc_size *= fbnic_bd_page_count(rxr); break; case FBNIC_QUEUE_BDQ_PPQ_TAIL: - rxq_size = fbn->ppq_size / FBNIC_BD_FRAG_COUNT; - desc_size *= FBNIC_BD_FRAG_COUNT; + rxq_size = fbn->ppq_size / fbnic_bd_page_count(rxr); + desc_size *= fbnic_bd_page_count(rxr); break; case FBNIC_QUEUE_RCQ_HEAD: rxq_size = fbn->rcq_size; @@ -2029,15 +2060,18 @@ free_sub0: static int fbnic_alloc_rx_qt_resources(struct fbnic_net *fbn, struct fbnic_napi_vector *nv, - struct fbnic_q_triad *qt) + struct fbnic_q_triad *qt, + u32 rx_page_size) { struct device *dev = fbn->netdev->dev.parent; int err; - err = fbnic_alloc_qt_page_pools(fbn, qt, qt->cmpl.q_idx); + err = fbnic_alloc_qt_page_pools(fbn, qt, qt->cmpl.q_idx, rx_page_size); if (err) return err; + fbnic_bdq_set_page_size(&qt->sub1, rx_page_size); + err = xdp_rxq_info_reg(&qt->xdp_rxq, fbn->netdev, qt->sub0.q_idx, nv->napi.napi_id); if (err) @@ -2098,7 +2132,11 @@ static int fbnic_alloc_nv_resources(struct fbnic_net *fbn, /* Allocate Rx Resources */ for (j = 0; j < nv->rxt_count; j++, i++) { - err = fbnic_alloc_rx_qt_resources(fbn, nv, &nv->qt[i]); + struct netdev_queue_config qcfg; + + netdev_queue_config(fbn->netdev, nv->qt[i].cmpl.q_idx, &qcfg); + err = fbnic_alloc_rx_qt_resources(fbn, nv, &nv->qt[i], + qcfg.rx_page_size); if (err) goto free_qt_resources; } @@ -2592,7 +2630,7 @@ static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) hpq->tail = 0; hpq->head = 0; - log_size = fls(hpq->size_mask) + ilog2(FBNIC_BD_FRAG_COUNT); + log_size = fls(hpq->size_mask) + hpq->bd_page_shift; /* Store descriptor ring address and size */ fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_BAL, lower_32_bits(hpq->dma)); @@ -2604,7 +2642,7 @@ static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) if (!ppq->size_mask) goto write_ctl; - log_size = fls(ppq->size_mask) + ilog2(FBNIC_BD_FRAG_COUNT); + log_size = fls(ppq->size_mask) + ppq->bd_page_shift; /* Add enabling of PPQ to BDQ control */ bdq_ctl |= FBNIC_QUEUE_BDQ_CTL_PPQ_ENABLE; @@ -2893,7 +2931,8 @@ static int fbnic_queue_mem_alloc(struct net_device *dev, struct fbnic_napi_vector *nv; if (!netif_running(dev)) - return fbnic_alloc_qt_page_pools(fbn, qt, idx); + return fbnic_alloc_qt_page_pools(fbn, qt, idx, + qcfg->rx_page_size); /* A failed PCIe recovery or resume can leave the datapath torn down * while netif_running() is still true. This ndo runs before @@ -2913,7 +2952,70 @@ static int fbnic_queue_mem_alloc(struct net_device *dev, fbnic_ring_init(&qt->cmpl, real->cmpl.doorbell, real->cmpl.q_idx, real->cmpl.flags); - return fbnic_alloc_rx_qt_resources(fbn, nv, qt); + return fbnic_alloc_rx_qt_resources(fbn, nv, qt, qcfg->rx_page_size); +} + +static void fbnic_default_qcfg(struct net_device *dev, + struct netdev_queue_config *qcfg) +{ + qcfg->rx_page_size = PAGE_SIZE; +} + +static int fbnic_validate_qcfg(struct net_device *dev, + struct netdev_queue_config *qcfg, + struct netlink_ext_ack *extack) +{ + u32 bd_page_count, ppq_entries, frag_count; + u32 rx_page_size = qcfg->rx_page_size; + u32 ppq_size; + + if (!qcfg->rx_jumbo_ring_size) { + NL_SET_ERR_MSG_MOD(extack, + "rx-jumbo ring size must be nonzero"); + return -EINVAL; + } + + ppq_size = roundup_pow_of_two(qcfg->rx_jumbo_ring_size); + + if (!is_power_of_2(rx_page_size)) { + NL_SET_ERR_MSG_MOD(extack, + "rx_page_size must be a power of 2"); + return -EINVAL; + } + + if (rx_page_size < FBNIC_BD_PAGE_SIZE) { + NL_SET_ERR_MSG_MOD(extack, + "rx_page_size must be at least 4K"); + return -EINVAL; + } + + /* Payload fragments occupy multiples of FBNIC_RX_PAYLD_ALIGN bytes. + * Keep at least one reference in the bias until fbnic_clean_bdq() + * observes a completion from a subsequent allocation. + */ + frag_count = rx_page_size / FBNIC_RX_PAYLD_ALIGN; + if (frag_count >= FBNIC_PAGECNT_BIAS_MAX) { + NL_SET_ERR_MSG_MOD(extack, + "rx_page_size can produce too many fragments"); + return -EINVAL; + } + + bd_page_count = rx_page_size / FBNIC_BD_PAGE_SIZE; + ppq_entries = ppq_size / bd_page_count; + /* The PPQ is sized in 4 KiB device pages. One software entry tracks + * each page-pool allocation. In addition to the unused entry for + * empty/full accounting, cleanup retains the current allocation + * until a completion identifies a subsequent allocation. A two-entry + * ring can only post one allocation and cannot make progress. + * Require at least four entries, since ring sizes are powers of two. + */ + if (ppq_entries < 4) { + NL_SET_ERR_MSG_MOD(extack, + "rx-jumbo ring size too small for rx_page_size"); + return -EINVAL; + } + + return 0; } static void fbnic_queue_mem_free(struct net_device *dev, void *qmem) @@ -3015,4 +3117,7 @@ const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops = { .ndo_queue_mem_free = fbnic_queue_mem_free, .ndo_queue_start = fbnic_queue_start, .ndo_queue_stop = fbnic_queue_stop, + .ndo_default_qcfg = fbnic_default_qcfg, + .ndo_validate_qcfg = fbnic_validate_qcfg, + .supported_params = QCFG_RX_PAGE_SIZE, }; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h index 66c287a5404a..6ee3cce3d942 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h @@ -65,6 +65,7 @@ struct fbnic_net; #define FBNIC_RX_HROOM \ (ALIGN(FBNIC_RX_TROOM + FBNIC_RX_HROOM_PAD, 128) - FBNIC_RX_TROOM) #define FBNIC_RX_PAD 0 +#define FBNIC_RX_PAYLD_ALIGN 128 #define FBNIC_RX_PAYLD_OFFSET 0 #define FBNIC_RX_PAYLD_PG_CL 0 @@ -131,6 +132,7 @@ struct fbnic_ring { u16 size_mask; /* Size of ring in descriptors - 1 */ u8 q_idx; /* Logical netdev ring index */ u8 flags; /* Ring flags (FBNIC_RING_F_*) */ + u8 bd_page_shift; /* BDQ: ilog2(page_size / 4096) */ u32 head, tail; /* Head/Tail of ring */ @@ -177,6 +179,11 @@ struct fbnic_napi_vector { extern const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops; +static inline u16 fbnic_bd_page_count(const struct fbnic_ring *bdq) +{ + return 1U << bdq->bd_page_shift; +} + netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev); netdev_features_t fbnic_features_check(struct sk_buff *skb, struct net_device *dev, diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index c4c9ce038611..c3a41fbd5426 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -1025,7 +1025,9 @@ struct kernel_ethtool_ts_info { * types should be set in @supported_coalesce_params. * Returns a negative error code or zero. * @get_ringparam: Report ring sizes - * @set_ringparam: Set ring sizes. Returns a negative error code or zero. + * @set_ringparam: Set ring sizes. The &struct ethtool_ringparam argument is + * also an output; drivers which normalize requested sizes must update it + * with the applied sizes. Returns a negative error code or zero. * @get_pause_stats: Report pause frame statistics. Drivers must not zero * statistics which they don't report. The stats structure is initialized * to ETHTOOL_STAT_NOT_SET indicating driver does not report statistics. diff --git a/include/net/netdev_queues.h b/include/net/netdev_queues.h index 70c9fe9e83cc..c5335e935b7d 100644 --- a/include/net/netdev_queues.h +++ b/include/net/netdev_queues.h @@ -4,18 +4,56 @@ #include +/** + * struct netdev_ring_config - accepted RX ring depth configuration + * @rx_pending: Size of the regular RX ring. + * @rx_mini_pending: Size of the RX mini ring. + * @rx_jumbo_pending: Size of the RX jumbo ring. + * + * This stores only persistent configuration values. Capability fields, + * such as max ring sizes, are reported by drivers but are not part of the + * accepted configuration. + * + * These values are only used for queue-configuration validation today. + * Drivers which normalize ring sizes must update the struct ethtool_ringparam + * passed to set_ringparam() with the applied sizes. + */ +struct netdev_ring_config { + u32 rx_pending; + u32 rx_mini_pending; + u32 rx_jumbo_pending; +}; + /** * struct netdev_config - queue-related configuration for a netdev * @hds_thresh: HDS Threshold value. * @hds_config: HDS value from userspace. + * @rings: Accepted RX ring depths. + * + * Direct values, such as @hds_thresh and @rings, hold the accepted + * configuration and always override callback-provided defaults, including + * when zero. Drivers which use @rings for queue rendering must initialize + * every RX ring depth before registering the netdev. */ struct netdev_config { u32 hds_thresh; u8 hds_config; + + struct netdev_ring_config rings; }; +/** + * struct netdev_queue_config - rendered configuration for an RX queue + * @rx_page_size: Size of one RX page-pool allocation. + * @rx_ring_size: Configured size of the regular RX ring. + * @rx_mini_ring_size: Configured size of the RX mini ring. + * @rx_jumbo_ring_size: Configured size of the RX jumbo ring. + */ struct netdev_queue_config { u32 rx_page_size; + u32 rx_ring_size; + u32 rx_mini_ring_size; + u32 rx_jumbo_ring_size; }; /* See the netdev.yaml spec for definition of each statistic */ @@ -139,16 +177,20 @@ enum { * @ndo_queue_get_dma_dev: Get dma device for zero-copy operations to be used * for this queue. Return NULL on error. * - * @ndo_default_qcfg: (Optional) Populate queue config struct with defaults. - * Queue config structs are passed to this helper before - * the user-requested settings are applied. + * @ndo_default_qcfg: (Optional) Populate queue config with defaults. Queue + * config structs are passed to this helper before the + * user-requested settings are applied. Ring depths from + * dev->cfg override these defaults. Drivers which consume + * them must initialize dev->cfg->rings before registering + * the netdev. * * @ndo_validate_qcfg: (Optional) Check if queue config is supported. * Called when configuration affecting a queue may be - * changing, either due to NIC-wide config, or config - * scoped to the queue at a specified index. - * When NIC-wide config is changed the callback will - * be invoked for all queues. + * changing. When NIC-wide config is changed the + * callback will be invoked for the defaults and all + * queue overrides. Drivers which normalize device-wide + * values when applying them must use the same + * normalization during validation. * * @ndo_queue_create: Create a new RX queue on a virtual device that will * be paired with a physical device's queue via leasing. diff --git a/net/core/dev.c b/net/core/dev.c index a8eb382f40ca..79be99b52f20 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -12243,10 +12243,8 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, if (!dev->ethtool) goto free_all; - dev->cfg = kzalloc_obj(*dev->cfg, GFP_KERNEL_ACCOUNT); - if (!dev->cfg) + if (netdev_alloc_config(dev)) goto free_all; - dev->cfg_pending = dev->cfg; dev->num_napi_configs = maxqs; napi_config_sz = array_size(maxqs, sizeof(*dev->napi_config)); @@ -12318,8 +12316,7 @@ void free_netdev(struct net_device *dev) return; } - WARN_ON(dev->cfg != dev->cfg_pending); - kfree(dev->cfg); + netdev_free_config(dev); kfree(dev->ethtool); netif_free_tx_queues(dev); netif_free_rx_queues(dev); diff --git a/net/core/dev.h b/net/core/dev.h index 51921623fc0a..4793a994f7f1 100644 --- a/net/core/dev.h +++ b/net/core/dev.h @@ -104,9 +104,16 @@ extern struct rw_semaphore dev_addr_sem; extern struct list_head net_todo_list; void netdev_run_todo(void); +int netdev_alloc_config(struct net_device *dev); +void __netdev_free_config(struct netdev_config *cfg); +void netdev_free_config(struct net_device *dev); +int netdev_reconfig_start(struct net_device *dev); + int netdev_queue_config_validate(struct net_device *dev, int rxq_idx, struct netdev_queue_config *qcfg, struct netlink_ext_ack *extack); +int netdev_queue_config_revalidate(struct net_device *dev, + struct netlink_ext_ack *extack); bool netif_rxq_has_mp(struct net_device *dev, unsigned int rxq_idx); bool netif_rxq_is_leased(struct net_device *dev, unsigned int rxq_idx); diff --git a/net/core/netdev_config.c b/net/core/netdev_config.c index f14af365d5cd..1975de42a60d 100644 --- a/net/core/netdev_config.c +++ b/net/core/netdev_config.c @@ -6,6 +6,43 @@ #include "dev.h" +int netdev_alloc_config(struct net_device *dev) +{ + struct netdev_config *cfg; + + cfg = kzalloc_obj(*dev->cfg, GFP_KERNEL_ACCOUNT); + if (!cfg) + return -ENOMEM; + + dev->cfg = cfg; + dev->cfg_pending = cfg; + return 0; +} + +void __netdev_free_config(struct netdev_config *cfg) +{ + kfree(cfg); +} + +void netdev_free_config(struct net_device *dev) +{ + WARN_ON(dev->cfg != dev->cfg_pending); + __netdev_free_config(dev->cfg); +} + +int netdev_reconfig_start(struct net_device *dev) +{ + struct netdev_config *cfg; + + WARN_ON(dev->cfg != dev->cfg_pending); + cfg = kmemdup(dev->cfg, sizeof(*dev->cfg), GFP_KERNEL_ACCOUNT); + if (!cfg) + return -ENOMEM; + + dev->cfg_pending = cfg; + return 0; +} + static int netdev_nop_validate_qcfg(struct net_device *dev, struct netdev_queue_config *qcfg, struct netlink_ext_ack *extack) @@ -13,6 +50,15 @@ static int netdev_nop_validate_qcfg(struct net_device *dev, return 0; } +static void netdev_qcfg_apply_dev(struct netdev_queue_config *qcfg, + const struct netdev_config *cfg) +{ + /* Device config overrides callback-provided fallbacks. */ + qcfg->rx_ring_size = cfg->rings.rx_pending; + qcfg->rx_mini_ring_size = cfg->rings.rx_mini_pending; + qcfg->rx_jumbo_ring_size = cfg->rings.rx_jumbo_pending; +} + static int __netdev_queue_config(struct net_device *dev, int rxq_idx, struct netdev_queue_config *qcfg, struct netlink_ext_ack *extack, @@ -33,6 +79,7 @@ static int __netdev_queue_config(struct net_device *dev, int rxq_idx, /* Get defaults from the driver, in case user config not set */ if (dev->queue_mgmt_ops->ndo_default_qcfg) dev->queue_mgmt_ops->ndo_default_qcfg(dev, qcfg); + netdev_qcfg_apply_dev(qcfg, dev->cfg_pending); err = validate_cb(dev, qcfg, extack); if (err) return err; @@ -54,9 +101,11 @@ static int __netdev_queue_config(struct net_device *dev, int rxq_idx, * @rxq_idx: index of the queue of interest * @qcfg: queue configuration struct (output) * - * Render the configuration for a given queue. This helper should be used - * by drivers which support queue configuration to retrieve config for - * a particular queue. + * Render the configuration for a given queue. During a configuration + * transaction this includes the proposed device-wide values in + * @dev->cfg_pending; otherwise @dev->cfg_pending points to the accepted + * configuration. This helper should be used by drivers which support queue + * configuration to retrieve config for a particular queue. * * @qcfg is an output parameter and is always fully initialized by this * function. Some values may not be set by the user, drivers may either @@ -76,3 +125,23 @@ int netdev_queue_config_validate(struct net_device *dev, int rxq_idx, { return __netdev_queue_config(dev, rxq_idx, qcfg, extack, true); } + +int netdev_queue_config_revalidate(struct net_device *dev, + struct netlink_ext_ack *extack) +{ + const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops; + struct netdev_queue_config qcfg; + unsigned int i; + int err; + + if (!qops || !qops->ndo_validate_qcfg) + return 0; + + for (i = 0; i < dev->real_num_rx_queues; i++) { + err = netdev_queue_config_validate(dev, i, &qcfg, extack); + if (err) + return err; + } + + return 0; +} diff --git a/net/ethtool/common.c b/net/ethtool/common.c index b4e766e60d38..ec67fbd6bb86 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -959,6 +959,14 @@ void ethtool_ringparam_get_cfg(struct net_device *dev, kparam->hds_thresh = dev->cfg->hds_thresh; } +void ethtool_ringparam_set_cfg(struct netdev_config *cfg, + const struct ethtool_ringparam *param) +{ + cfg->rings.rx_pending = param->rx_pending; + cfg->rings.rx_mini_pending = param->rx_mini_pending; + cfg->rings.rx_jumbo_pending = param->rx_jumbo_pending; +} + static void ethtool_init_tsinfo(struct kernel_ethtool_ts_info *info) { memset(info, 0, sizeof(*info)); diff --git a/net/ethtool/common.h b/net/ethtool/common.h index ae32e7fdb563..b062e99e89db 100644 --- a/net/ethtool/common.h +++ b/net/ethtool/common.h @@ -53,6 +53,8 @@ void ethtool_ringparam_get_cfg(struct net_device *dev, struct ethtool_ringparam *param, struct kernel_ethtool_ringparam *kparam, struct netlink_ext_ack *extack); +void ethtool_ringparam_set_cfg(struct netdev_config *cfg, + const struct ethtool_ringparam *param); int ethtool_get_rx_ring_count(struct net_device *dev); diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 2b449d08ddcf..27b0e2e275dc 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -35,6 +35,7 @@ #include #include "common.h" +#include "../core/dev.h" /* State held across locks and calls for commands which have devlink fallback */ struct ethtool_devlink_compat { @@ -2230,10 +2231,29 @@ static int ethtool_set_ringparam(struct net_device *dev, void __user *useraddr) ringparam.tx_pending > max.tx_max_pending) return -EINVAL; + ret = netdev_reconfig_start(dev); + if (ret) + return ret; + + ethtool_ringparam_set_cfg(dev->cfg_pending, &ringparam); + + ret = netdev_queue_config_revalidate(dev, NULL); + if (ret) + goto out_free_cfg; + ret = dev->ethtool_ops->set_ringparam(dev, &ringparam, &kernel_ringparam, NULL); - if (!ret) - ethtool_notify(dev, ETHTOOL_MSG_RINGS_NTF); + if (ret) + goto out_free_cfg; + + /* Capture ring depth adjustments reported by the driver. */ + ethtool_ringparam_set_cfg(dev->cfg_pending, &ringparam); + swap(dev->cfg, dev->cfg_pending); + ethtool_notify(dev, ETHTOOL_MSG_RINGS_NTF); + +out_free_cfg: + __netdev_free_config(dev->cfg_pending); + dev->cfg_pending = dev->cfg; return ret; } diff --git a/net/ethtool/netlink.c b/net/ethtool/netlink.c index 1af395b54330..383e911f50f7 100644 --- a/net/ethtool/netlink.c +++ b/net/ethtool/netlink.c @@ -11,6 +11,8 @@ #include "module_fw.h" #include "netlink.h" +#include "../core/dev.h" + static struct genl_family ethtool_genl_family; static bool ethnl_ok __read_mostly; @@ -934,12 +936,9 @@ static int ethnl_default_set_doit(struct sk_buff *skb, struct genl_info *info) if (need_rtnl) rtnl_lock(); netdev_lock_ops(dev); - dev->cfg_pending = kmemdup(dev->cfg, sizeof(*dev->cfg), - GFP_KERNEL_ACCOUNT); - if (!dev->cfg_pending) { - ret = -ENOMEM; - goto out_tie_cfg; - } + ret = netdev_reconfig_start(dev); + if (ret) + goto out_unlock; ret = ethnl_ops_begin(dev); if (ret < 0) @@ -958,9 +957,9 @@ static int ethnl_default_set_doit(struct sk_buff *skb, struct genl_info *info) out_ops: ethnl_ops_complete(dev); out_free_cfg: - kfree(dev->cfg_pending); -out_tie_cfg: + __netdev_free_config(dev->cfg_pending); dev->cfg_pending = dev->cfg; +out_unlock: netdev_unlock_ops(dev); if (need_rtnl) rtnl_unlock(); diff --git a/net/ethtool/rings.c b/net/ethtool/rings.c index 9054c89c5d7b..e3810c0320e3 100644 --- a/net/ethtool/rings.c +++ b/net/ethtool/rings.c @@ -4,6 +4,7 @@ #include "common.h" #include "netlink.h" +#include "../core/dev.h" struct rings_req_info { struct ethnl_req_info base; @@ -299,10 +300,20 @@ ethnl_set_rings(struct ethnl_req_info *req_info, struct genl_info *info) dev->cfg_pending->hds_config = kernel_ringparam.tcp_data_split; dev->cfg_pending->hds_thresh = kernel_ringparam.hds_thresh; + ethtool_ringparam_set_cfg(dev->cfg_pending, &ringparam); + + ret = netdev_queue_config_revalidate(dev, info->extack); + if (ret) + return ret; ret = dev->ethtool_ops->set_ringparam(dev, &ringparam, &kernel_ringparam, info->extack); - return ret < 0 ? ret : 1; + if (ret < 0) + return ret; + + /* Capture ring depth adjustments reported by the driver. */ + ethtool_ringparam_set_cfg(dev->cfg_pending, &ringparam); + return 1; } const struct ethnl_request_ops ethnl_rings_request_ops = { diff --git a/tools/testing/selftests/drivers/net/hw/iou-zcrx.py b/tools/testing/selftests/drivers/net/hw/iou-zcrx.py index c833535d8a03..5047fdb56005 100755 --- a/tools/testing/selftests/drivers/net/hw/iou-zcrx.py +++ b/tools/testing/selftests/drivers/net/hw/iou-zcrx.py @@ -221,9 +221,13 @@ def test_zcrx_large_chunks(cfg) -> None: single(cfg) page_size = resource.getpagesize() + mtu = cfg.dev["mtu"] nr_pages = 2 + while nr_pages * page_size <= 2 * mtu: + nr_pages *= 2 rx_buf_len = nr_pages * page_size - rx_cmd = f"{cfg.bin_local} -s -p {cfg.port} -i {cfg.ifname} -q {cfg.target} -x {nr_pages}" + rx_cmd = (f"{cfg.bin_local} -s -p {cfg.port} -i {cfg.ifname} " + f"-q {cfg.target} -x {nr_pages}") tx_cmd = f"{cfg.bin_remote} -c -h {cfg.addr_v['6']} -p {cfg.port} -l 12840" probe = cmd(rx_cmd + " -d", fail=False)