From f2109e90a5aaedb10b4ac7927ec074820a452d57 Mon Sep 17 00:00:00 2001 From: Willem de Bruijn Date: Thu, 10 Sep 2026 13:10:20 -0400 Subject: [PATCH 1/7] net: rtnetlink: add pacing_offload attribute to net_device Add a new flag to administratively control pacing offload. The feature is disabled by default. That does not cause regressions, as no driver advertises max_pacing_offload_horizon yet. Also add NLA_REJECT for max_pacing_offload_horizon, in line with other such read-only members of link-attrs. Both fields can be read with ynl --family rt-link --do getlink \ --json '{"ifname": "eth0"}' | grep pacing And pacing offload enabled with ynl --family rt-link --do setlink \ --json '{"ifname": "eth0", "pacing-offload": 1}' Signed-off-by: Willem de Bruijn Link: https://patch.msgid.link/20260910171131.2532487-2-willemdebruijn.kernel@gmail.com Signed-off-by: Jakub Kicinski --- Documentation/netlink/specs/rt-link.yaml | 5 +++++ .../networking/net_cachelines/net_device.rst | 1 + include/linux/netdevice.h | 2 ++ include/uapi/linux/if_link.h | 1 + net/core/rtnetlink.c | 22 +++++++++++++++++++ 5 files changed, 31 insertions(+) diff --git a/Documentation/netlink/specs/rt-link.yaml b/Documentation/netlink/specs/rt-link.yaml index 5217d0bb4a3a..7a72cd1b7e1e 100644 --- a/Documentation/netlink/specs/rt-link.yaml +++ b/Documentation/netlink/specs/rt-link.yaml @@ -1091,6 +1091,10 @@ attribute-sets: - name: tailroom type: u16 + - + name: pacing-offload + type: u32 + doc: Enable EDT pacing offload (0 - disabled, 1 - enabled). - name: prop-list-link-attrs subset-of: link-attrs @@ -2559,6 +2563,7 @@ operations: - devlink-port - gso-ipv4-max-size - gro-ipv4-max-size + - pacing-offload dump: request: value: 18 diff --git a/Documentation/networking/net_cachelines/net_device.rst b/Documentation/networking/net_cachelines/net_device.rst index 512f6d6fa3d8..8eceaa80b686 100644 --- a/Documentation/networking/net_cachelines/net_device.rst +++ b/Documentation/networking/net_cachelines/net_device.rst @@ -11,6 +11,7 @@ Type Name fastpath_tx_acce unsigned_long:32 priv_flags read_mostly __dev_queue_xmit(tx) unsigned_long:1 lltx read_mostly HARD_TX_LOCK,HARD_TX_TRYLOCK,HARD_TX_UNLOCK(tx) unsigned_long:2 netmem_tx:2; read_mostly +unsigned_long:1 pacing_offload read_mostly sch_fq char name[16] struct netdev_name_node* name_node struct dev_ifalias* ifalias diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 707b2e51c2b9..1f0710eef185 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1840,6 +1840,7 @@ enum netdev_reg_state { * drivers. Mainly used by logical interfaces, such as * bonding and tunnels * @netmem_tx: device netmem TX mode + * @pacing_offload: enable EDT pacing offload. * * @name: This is the first field of the "visible" part of this structure * (i.e. as seen by users in the "Space.c" file). It is the name @@ -2170,6 +2171,7 @@ struct net_device { unsigned long priv_flags:32; unsigned long lltx:1; unsigned long netmem_tx:2; + unsigned long pacing_offload:1; ); const struct net_device_ops *netdev_ops; const struct header_ops *header_ops; diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index 43cecca49f01..245b36204525 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -381,6 +381,7 @@ enum { IFLA_NETNS_IMMUTABLE, IFLA_HEADROOM, IFLA_TAILROOM, + IFLA_PACING_OFFLOAD, __IFLA_MAX }; diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index be9d1625bac3..e3444fd24061 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -1396,6 +1396,7 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev, + rtnl_devlink_port_size(dev) + rtnl_dpll_pin_size() + nla_total_size(8) /* IFLA_MAX_PACING_OFFLOAD_HORIZON */ + + nla_total_size(4) /* IFLA_PACING_OFFLOAD */ + nla_total_size(2) /* IFLA_HEADROOM */ + nla_total_size(2) /* IFLA_TAILROOM */ + rtnl_dev_parent_size(dev) @@ -2176,6 +2177,8 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, READ_ONCE(dev->tso_max_segs)) || nla_put_uint(skb, IFLA_MAX_PACING_OFFLOAD_HORIZON, READ_ONCE(dev->max_pacing_offload_horizon)) || + nla_put_u32(skb, IFLA_PACING_OFFLOAD, + dev->pacing_offload) || #ifdef CONFIG_RPS nla_put_u32(skb, IFLA_NUM_RX_QUEUES, READ_ONCE(dev->num_rx_queues)) || @@ -2349,9 +2352,11 @@ static const struct nla_policy ifla_policy[IFLA_MAX+1] = { [IFLA_ALLMULTI] = { .type = NLA_REJECT }, [IFLA_GSO_IPV4_MAX_SIZE] = NLA_POLICY_MIN(NLA_U32, MAX_TCP_HEADER + 1), [IFLA_GRO_IPV4_MAX_SIZE] = { .type = NLA_U32 }, + [IFLA_MAX_PACING_OFFLOAD_HORIZON] = { .type = NLA_REJECT }, [IFLA_NETNS_IMMUTABLE] = { .type = NLA_REJECT }, [IFLA_HEADROOM] = { .type = NLA_REJECT }, [IFLA_TAILROOM] = { .type = NLA_REJECT }, + [IFLA_PACING_OFFLOAD] = NLA_POLICY_MAX(NLA_U32, 1), }; static const struct nla_policy ifla_info_policy[IFLA_INFO_MAX+1] = { @@ -2825,6 +2830,14 @@ static int validate_linkmsg(struct net_device *dev, struct nlattr *tb[], return -EINVAL; } + if (tb[IFLA_PACING_OFFLOAD]) { + if (nla_get_u32(tb[IFLA_PACING_OFFLOAD]) && + !dev->max_pacing_offload_horizon) { + NL_SET_ERR_MSG(extack, "pacing offload not supported by device"); + return -EOPNOTSUPP; + } + } + if (tb[IFLA_AF_SPEC]) { struct nlattr *af; int rem, err; @@ -3342,6 +3355,15 @@ static int do_setlink(const struct sk_buff *skb, struct net_device *dev, } } + if (tb[IFLA_PACING_OFFLOAD]) { + bool val = nla_get_u32(tb[IFLA_PACING_OFFLOAD]); + + if (dev->pacing_offload != val) { + dev->pacing_offload = val; + status |= DO_SETLINK_MODIFIED; + } + } + if (tb[IFLA_OPERSTATE]) set_operstate(dev, nla_get_u8(tb[IFLA_OPERSTATE])); From 8036cbf026b512378a024fadb1c452356f01ed8b Mon Sep 17 00:00:00 2001 From: Willem de Bruijn Date: Thu, 10 Sep 2026 13:10:21 -0400 Subject: [PATCH 2/7] net_sched: sch_fq: check device pacing offload Network devices can modify pacing offload administratively or device renegotiation. In FQ, only offload if the feature is enabled and within the supported device horizon. Signed-off-by: Willem de Bruijn Link: https://patch.msgid.link/20260910171131.2532487-3-willemdebruijn.kernel@gmail.com Signed-off-by: Jakub Kicinski --- net/sched/sch_fq.c | 53 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index 35f940b2205d..dfc7f98b600f 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -301,6 +301,26 @@ static void fq_gc(struct fq_sched_data *q, q->stat_gc_flows += fcnt; } +static u64 fq_offload_horizon(const struct Qdisc *sch, + const struct fq_sched_data *q) +{ + const struct net_device *dev; + u64 offload_horizon; + + offload_horizon = READ_ONCE(q->offload_horizon); + if (!offload_horizon) + return 0; + + dev = qdisc_dev(sch); + if (!dev->pacing_offload) + return 0; + + if (offload_horizon > READ_ONCE(dev->max_pacing_offload_horizon)) + return 0; + + return offload_horizon; +} + /* Fast path can be used if : * 1) Packet tstamp is in the past, or within the pacing offload horizon. * 2) FQ qlen == 0 OR @@ -312,12 +332,12 @@ static void fq_gc(struct fq_sched_data *q, * FQ can not use generic TCQ_F_CAN_BYPASS infrastructure. */ static bool fq_fastpath_check(const struct Qdisc *sch, struct sk_buff *skb, - u64 now) + u64 now, u64 offload_horizon) { const struct fq_sched_data *q = qdisc_priv(sch); const struct sock *sk; - if (fq_skb_cb(skb)->time_to_send > now + q->offload_horizon) + if (fq_skb_cb(skb)->time_to_send > now + offload_horizon) return false; if (sch->q.qlen != 0) { @@ -338,7 +358,7 @@ static bool fq_fastpath_check(const struct Qdisc *sch, struct sk_buff *skb, /* Ordering invariants fall apart if some delayed flows * are ready but we haven't serviced them, yet. */ - if (q->time_next_delayed_flow <= now + q->offload_horizon) + if (q->time_next_delayed_flow <= now + offload_horizon) return false; } @@ -357,6 +377,7 @@ static struct fq_flow *fq_classify(struct Qdisc *sch, struct sk_buff *skb, u64 now) { struct fq_sched_data *q = qdisc_priv(sch); + u64 offload_horizon = fq_offload_horizon(sch, q); struct rb_node **p, *parent; struct sock *sk = skb->sk; struct rb_root *root; @@ -393,12 +414,13 @@ static struct fq_flow *fq_classify(struct Qdisc *sch, struct sk_buff *skb, sk = (struct sock *)((hash << 1) | 1UL); } - if (fq_fastpath_check(sch, skb, now)) { + if (fq_fastpath_check(sch, skb, now, offload_horizon)) { q->internal.stat_fastpath_packets++; if (skb->sk == sk && q->rate_enable && READ_ONCE(sk->sk_pacing_status) != SK_PACING_FQ) smp_store_release(&sk->sk_pacing_status, SK_PACING_FQ); + return &q->internal; } @@ -661,12 +683,13 @@ static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch, return NET_XMIT_SUCCESS; } -static void fq_check_throttled(struct fq_sched_data *q, u64 now) +static void fq_check_throttled(struct fq_sched_data *q, u64 now, + u64 offload_horizon) { unsigned long sample; struct rb_node *p; - if (q->time_next_delayed_flow > now + q->offload_horizon) + if (q->time_next_delayed_flow > now + offload_horizon) return; /* Update unthrottle latency EWMA. @@ -677,7 +700,7 @@ static void fq_check_throttled(struct fq_sched_data *q, u64 now) q->unthrottle_latency_ns -= q->unthrottle_latency_ns >> 3; q->unthrottle_latency_ns += sample >> 3; } - now += q->offload_horizon; + now += offload_horizon; q->time_next_delayed_flow = ~0ULL; while ((p = rb_first(&q->delayed)) != NULL) { @@ -705,6 +728,7 @@ static struct fq_flow_head *fq_pband_head_select(struct fq_perband_flows *pband) static struct sk_buff *fq_dequeue(struct Qdisc *sch) { struct fq_sched_data *q = qdisc_priv(sch); + u64 offload_horizon = fq_offload_horizon(sch, q); struct fq_perband_flows *pband; struct fq_flow_head *head; struct sk_buff *skb; @@ -725,7 +749,7 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch) } now = ktime_get_ns(); - fq_check_throttled(q, now); + fq_check_throttled(q, now, offload_horizon); retry = 0; pband = &q->band_flows[q->band_nr]; begin: @@ -761,7 +785,7 @@ begin: u64 time_next_packet = max_t(u64, fq_skb_cb(skb)->time_to_send, f->time_next_packet); - if (now + q->offload_horizon < time_next_packet) { + if (now + offload_horizon < time_next_packet) { head->first = f->next; f->time_next_packet = time_next_packet; fq_flow_set_throttled(q, f); @@ -836,6 +860,7 @@ begin: } f->time_next_packet = now + len; } + out: return skb; } @@ -1179,11 +1204,15 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt, u64 offload_horizon = (u64)NSEC_PER_USEC * nla_get_u32(tb[TCA_FQ_OFFLOAD_HORIZON]); - if (offload_horizon <= qdisc_dev(sch)->max_pacing_offload_horizon) { - WRITE_ONCE(q->offload_horizon, offload_horizon); - } else { + if (offload_horizon && !qdisc_dev(sch)->pacing_offload) { + NL_SET_ERR_MSG_MOD(extack, "device pacing offload is disabled"); + err = -EINVAL; + } else if (offload_horizon > + qdisc_dev(sch)->max_pacing_offload_horizon) { NL_SET_ERR_MSG_MOD(extack, "invalid offload_horizon"); err = -EINVAL; + } else { + WRITE_ONCE(q->offload_horizon, offload_horizon); } } if (!err) { From e628a85c69ab01d81801fd0b0e51a43b3b999320 Mon Sep 17 00:00:00 2001 From: Willem de Bruijn Date: Thu, 10 Sep 2026 13:10:22 -0400 Subject: [PATCH 3/7] net_sched: sch_fq: clear past skb->tstamp if offloading pacing When hardware offload is enabled, FQ will forward packets to the netdevice for pacing. The device has to test that skb->tstamp is in the future. Avoid this cost for packets whose txtime has already passed, by clearing skb->tstamp. Also clear slightly into the future, for EDT timestamps that are so close to now that they fall within a reasonable normal Tx latency. This slack is set to 400 nsec. Signed-off-by: Willem de Bruijn Link: https://patch.msgid.link/20260910171131.2532487-4-willemdebruijn.kernel@gmail.com Signed-off-by: Jakub Kicinski --- net/sched/sch_fq.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index dfc7f98b600f..a282812c192e 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -159,6 +159,9 @@ struct fq_sched_data { u64 stat_allocation_errors; }; +/* EDT timestamps to clear beyond now. */ +static const int fq_offload_slack_ns = 400; + /* return the i-th 2-bit value ("crumb") */ static u8 fq_prio2band(const u8 *prio2band, unsigned int prio) { @@ -421,6 +424,10 @@ static struct fq_flow *fq_classify(struct Qdisc *sch, struct sk_buff *skb, smp_store_release(&sk->sk_pacing_status, SK_PACING_FQ); + if (offload_horizon && + fq_skb_cb(skb)->time_to_send <= now + fq_offload_slack_ns) + skb_set_delivery_time(skb, 0, SKB_CLOCK_REALTIME); + return &q->internal; } @@ -731,6 +738,7 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch) u64 offload_horizon = fq_offload_horizon(sch, q); struct fq_perband_flows *pband; struct fq_flow_head *head; + u64 time_next_packet = 0; struct sk_buff *skb; struct fq_flow *f; unsigned long rate; @@ -745,7 +753,7 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch) if (skb) { q->internal.qlen--; fq_dequeue_skb(sch, &q->internal, skb); - goto out; + return skb; } now = ktime_get_ns(); @@ -782,8 +790,8 @@ begin: skb = fq_peek(f); if (skb) { - u64 time_next_packet = max_t(u64, fq_skb_cb(skb)->time_to_send, - f->time_next_packet); + time_next_packet = max_t(u64, fq_skb_cb(skb)->time_to_send, + f->time_next_packet); if (now + offload_horizon < time_next_packet) { head->first = f->next; @@ -862,6 +870,10 @@ begin: } out: + if (offload_horizon && + time_next_packet && time_next_packet <= now + fq_offload_slack_ns) + skb_set_delivery_time(skb, 0, SKB_CLOCK_REALTIME); + return skb; } From 4aa8b5cadeb3955a30d696d20d8067fdec0d4b4c Mon Sep 17 00:00:00 2001 From: Willem de Bruijn Date: Thu, 10 Sep 2026 13:10:23 -0400 Subject: [PATCH 4/7] idpf: support pacing offload If skb->tstamp is in the future, program this future delivery txtime in the transmit descriptor. TCP pacing offload is only offloaded if SK_PACING_FQ is negotiated and the FQ offload_horizon is configured. But device support for pacing offload must be more robust: it can also be reached through SO_TXTIME. Bounds check txtime. Only packets with timestamp between now and the horizon (max_pacing_offload_horizon) are offloaded when pacing offload is enabled on the device via pacing_offload. Negotiate the feature with the device using virtchnl. Support is conditional on - splitq mode, where tx and tx completion queues are separate, so completions can be returned out of order. - flow scheduling mode, where completions can arrive out of order. - PTP to ensure the NIC clock is synced to CLOCK_TAI. These features are negotiated per adapter, but expect all vports to uniformly request splitq (req_[rt]x_splitq) and flow scheduling (flow_sch_en) when available. Packets beyond the horizon are sent immediately with the overflow bit set. On device reset, EDT capabilities are re-negotiated with firmware. If re-negotiation succeeds, dev->max_pacing_offload_horizon is refreshed. If pacing offload is no longer supported, dev->max_pacing_offload_horizon is set to 0. Must not be called from netpoll due to ktime_get. But netpoll does not generate packets with EDT, so no explicit test is needed. Do not fail device initialization on EDT init error. Log an error, but continue without EDT, similar to PTP. Cc: Tony Nguyen Cc: Przemek Kitszel Cc: Joshua A Hay Cc: intel-wired-lan@lists.osuosl.org Cc: Aleksandr Loktionov Signed-off-by: Willem de Bruijn Link: https://patch.msgid.link/20260910171131.2532487-5-willemdebruijn.kernel@gmail.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/intel/idpf/idpf.h | 12 ++++ drivers/net/ethernet/intel/idpf/idpf_lib.c | 19 +++++ drivers/net/ethernet/intel/idpf/idpf_txrx.c | 69 ++++++++++++++++++- drivers/net/ethernet/intel/idpf/idpf_txrx.h | 8 ++- .../net/ethernet/intel/idpf/idpf_virtchnl.c | 60 +++++++++++++++- 5 files changed, 165 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h index 470bc23c844c..f214023095ee 100644 --- a/drivers/net/ethernet/intel/idpf/idpf.h +++ b/drivers/net/ethernet/intel/idpf/idpf.h @@ -604,6 +604,16 @@ struct idpf_vport_config { DECLARE_BITMAP(flags, IDPF_VPORT_CONFIG_FLAGS_NBITS); }; +/** + * struct idpf_edt_caps_ilog2 - Host parsed EDT capabilities. + * @time_horizon_ns: Total time window in nanoseconds. + * @tstamp_granularity_pow2: Log2 of timestamp granularity in nanoseconds. + */ +struct idpf_edt_caps_ilog2 { + u32 time_horizon_ns; + u8 tstamp_granularity_pow2; +}; + #define idpf_for_each_vport(adapter, iter) \ for (struct idpf_vport **__##iter = &(adapter)->vports[0], \ *iter = (adapter)->max_vports ? *__##iter : NULL; \ @@ -657,6 +667,7 @@ struct idpf_vport_config { * @stats_task: Periodic statistics retrieval task * @stats_wq: Workqueue for statistics task * @caps: Negotiated capabilities with device + * @edt_caps: Negotiated EDT capabilities with device * @dev_ops: See idpf_dev_ops * @cdev_info: IDC core device info pointer * @num_vfs: Number of allocated VFs through sysfs. PF does not directly talk @@ -720,6 +731,7 @@ struct idpf_adapter { struct delayed_work stats_task; struct workqueue_struct *stats_wq; struct virtchnl2_get_capabilities caps; + struct idpf_edt_caps_ilog2 edt_caps; struct idpf_dev_ops dev_ops; struct iidc_rdma_core_dev_info *cdev_info; diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index 827c795afcb6..cd1b173e3b27 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -817,6 +817,21 @@ static void idpf_attach_and_open(struct idpf_adapter *adapter) } } +static void idpf_vport_set_pacing_offload(struct idpf_vport *vport, + struct net_device *netdev) +{ + struct idpf_adapter *adapter = vport->adapter; + u32 max_horizon = 0; + + if (adapter->edt_caps.time_horizon_ns && + idpf_is_queue_model_split(vport->dflt_qv_rsrc.txq_model) && + !idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, + VIRTCHNL2_CAP_SPLITQ_QSCHED)) + max_horizon = adapter->edt_caps.time_horizon_ns; + + WRITE_ONCE(netdev->max_pacing_offload_horizon, max_horizon); +} + /** * idpf_cfg_netdev - Allocate, configure and register a netdev * @vport: main vport structure @@ -850,6 +865,8 @@ static int idpf_cfg_netdev(struct idpf_vport *vport) np->max_tx_hdr_size = idpf_get_max_tx_hdr_size(adapter); vport->netdev = netdev; + idpf_vport_set_pacing_offload(vport, netdev); + return idpf_init_mac_addr(vport, netdev); } @@ -890,6 +907,8 @@ static int idpf_cfg_netdev(struct idpf_vport *vport) netdev->min_mtu = ETH_MIN_MTU; netdev->max_mtu = vport->max_mtu; + idpf_vport_set_pacing_offload(vport, netdev); + dflt_features = NETIF_F_SG | NETIF_F_HIGHDMA; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 4311ffa30bb1..215a62161a04 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -1742,6 +1742,7 @@ static int idpf_txq_group_alloc(struct idpf_vport *vport, q->desc_count = rsrc->txq_desc_count; q->tx_max_bufs = idpf_get_max_tx_bufs(adapter); q->tx_min_pkt_len = idpf_get_min_tx_pkt_len(adapter); + q->ts_gran_pow2 = adapter->edt_caps.tstamp_granularity_pow2; q->netdev = vport->netdev; q->txq_grp = tx_qgrp; q->rel_q_id = j; @@ -2408,7 +2409,12 @@ void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc, struct idpf_tx_splitq_params *params, u16 td_cmd, u16 size) { - *(__le32 *)&desc->flow.qw1.cmd_dtype = cpu_to_le32((u8)(params->dtype | td_cmd)); + desc->flow.qw1.cmd_dtype = (u8)(params->dtype | td_cmd); + + desc->flow.qw1.ts[0] = params->offload.desc_ts[0]; + desc->flow.qw1.ts[1] = params->offload.desc_ts[1]; + desc->flow.qw1.ts[2] = params->offload.desc_ts[2]; + desc->flow.qw1.rxr_bufsize = cpu_to_le16((u16)size); desc->flow.qw1.compl_tag = cpu_to_le16(params->compl_tag); } @@ -3011,6 +3017,63 @@ static bool idpf_tx_splitq_need_re(struct idpf_tx_queue *tx_q) return gap >= IDPF_TX_SPLITQ_RE_MIN_GAP; } +static void idpf_tx_splitq_set_txtime(const struct sk_buff *skb, + const struct idpf_tx_queue *tx_q, + struct idpf_tx_splitq_params *tx_params) +{ + const int offload_slack_ns = 400; + u64 ts, now, horizon; + + if (!tx_q->netdev->pacing_offload) + return; + + horizon = READ_ONCE(tx_q->netdev->max_pacing_offload_horizon); + if (!horizon) + return; + + switch (skb->tstamp_type) { + case SKB_CLOCK_REALTIME: + ts = ktime_to_ns(ktime_add(skb->tstamp, + ktime_mono_to_any(0, TK_OFFS_TAI) - + ktime_mono_to_any(0, TK_OFFS_REAL))); + break; + case SKB_CLOCK_MONOTONIC: + ts = ktime_to_ns(ktime_mono_to_any(skb->tstamp, TK_OFFS_TAI)); + break; + case SKB_CLOCK_TAI: + ts = ktime_to_ns(skb->tstamp); + break; + default: + WARN_ON_ONCE(1); + return; + } + + now = ktime_get_clocktai_ns(); + if (ts < now + offload_slack_ns) + return; + + /* beyond offload horizon? set overflow bit only */ + if (ts > now + horizon) { + tx_params->offload.desc_ts[2] = + IDPF_TXD_FLOW_SCH_HORIZON_OVERFLOW_M; + return; + } + + ts >>= tx_q->ts_gran_pow2; + + /* 0 is valid 23b timestamp, but also means field unset. + * Increase by one to avoid this case + */ + if ((ts & 0x7fffff) == 0) { + tx_params->offload.desc_ts[0] = 1; + return; + } + + tx_params->offload.desc_ts[0] = ts & 0xff; + tx_params->offload.desc_ts[1] = (ts >> 8) & 0xff; + tx_params->offload.desc_ts[2] = ((ts >> 16) & 0x7f); +} + /** * idpf_tx_splitq_frame - Sends buffer on Tx ring using flex descriptors * @skb: send buffer @@ -3097,6 +3160,10 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_FLOW_SCHE; tx_params.eop_cmd = IDPF_TXD_FLEX_FLOW_CMD_EOP; + + if (skb->tstamp) + idpf_tx_splitq_set_txtime(skb, tx_q, &tx_params); + /* Set the RE bit periodically to "clean" the descriptor ring */ if (idpf_tx_splitq_need_re(tx_q)) { tx_params.eop_cmd |= IDPF_TXD_FLEX_FLOW_CMD_RE; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 93547597efd2..fe0c913f9bb9 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -161,6 +161,7 @@ union idpf_tx_flex_desc { * @tso_segs: Number of segments to be sent * @tso_hdr_len: Length of headers to be duplicated * @td_cmd: Command field to be inserted into descriptor + * @desc_ts: Flow scheduling offload timestamp */ struct idpf_tx_offload_params { u32 tx_flags; @@ -174,6 +175,7 @@ struct idpf_tx_offload_params { u16 tso_hdr_len; u16 td_cmd; + u8 desc_ts[3]; }; /** @@ -608,6 +610,7 @@ libeth_cacheline_set_assert(struct idpf_rx_queue, * hot path TX pointers stored in vport. Used in both singleq/splitq. * @desc_count: Number of descriptors * @tx_min_pkt_len: Min supported packet length + * @ts_gran_pow2: Txtime timestamp granularity in nanoseconds (log2). * @thresh: XDP queue cleaning threshold * @netdev: &net_device corresponding to this queue * @next_to_use: Next descriptor to use @@ -666,7 +669,10 @@ struct idpf_tx_queue { u16 desc_count; union { - u16 tx_min_pkt_len; + struct { + u16 tx_min_pkt_len; + u8 ts_gran_pow2; + }; u32 thresh; }; diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c index 1caf52706973..928de868fa62 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c @@ -3,6 +3,7 @@ #include #include +#include #include #include "idpf.h" @@ -553,7 +554,8 @@ static int idpf_send_get_caps_msg(struct idpf_adapter *adapter) VIRTCHNL2_CAP_SPLITQ_QSCHED | VIRTCHNL2_CAP_PROMISC | VIRTCHNL2_CAP_LOOPBACK | - VIRTCHNL2_CAP_PTP); + VIRTCHNL2_CAP_PTP | + VIRTCHNL2_CAP_EDT); err = idpf_send_mb_msg_stack(adapter, &xn_params, &caps); if (err) @@ -573,6 +575,54 @@ free_rx_buf: return err; } +/** + * idpf_send_get_edt_caps_msg - Send virtchnl get EDT caps msg + * @adapter: Driver specific private struct + * + * Return: 0 on success or error code on failure. + */ +static int idpf_send_get_edt_caps_msg(struct idpf_adapter *adapter) +{ + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_GET_EDT_CAPS, + }; + struct virtchnl2_edt_caps caps = {}; + u64 gran_ns, horizon_ns; + int err; + + err = idpf_send_mb_msg_stack(adapter, &xn_params, &caps); + if (err) + return err; + + if (xn_params.recv_mem.iov_len < sizeof(caps)) { + err = -EIO; + goto free_rx_buf; + } + + memcpy(&caps, xn_params.recv_mem.iov_base, sizeof(caps)); + horizon_ns = le64_to_cpu(caps.time_horizon_ns); + gran_ns = le64_to_cpu(caps.tstamp_granularity_ns); + if (horizon_ns > U32_MAX) { + dev_warn(&adapter->pdev->dev, "EDT horizon exceeds U32\n"); + err = -EINVAL; + goto free_rx_buf; + } + if (!gran_ns || !is_power_of_2(gran_ns)) { + dev_warn(&adapter->pdev->dev, "Invalid EDT granularity\n"); + err = -EINVAL; + goto free_rx_buf; + } + + adapter->edt_caps.time_horizon_ns = horizon_ns; + adapter->edt_caps.tstamp_granularity_pow2 = ilog2(gran_ns); + +free_rx_buf: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + + return err; +} + /** * idpf_mmio_region_non_static - Check if region is not static * @mmio_info: PCI resources info @@ -3085,6 +3135,14 @@ restart: } } + memset(&adapter->edt_caps, 0, sizeof(adapter->edt_caps)); + if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_EDT)) { + err = idpf_send_get_edt_caps_msg(adapter); + if (err) + dev_err(&adapter->pdev->dev, + "EDT init failed, err=%d\n", err); + } + pci_sriov_set_totalvfs(adapter->pdev, idpf_get_max_vfs(adapter)); num_max_vports = idpf_get_max_vports(adapter); adapter->vports = kzalloc_objs(*adapter->vports, num_max_vports); From 2d5c630e6cd52f2ee53167bf2d84220c22b5a480 Mon Sep 17 00:00:00 2001 From: Willem de Bruijn Date: Thu, 10 Sep 2026 13:10:24 -0400 Subject: [PATCH 5/7] selftests: drv-net: refactor so_txtime errqueue handling Prepare error queue handling for upcoming SO_EE_ORIGIN_TIMESTAMPING messages in the next patch in this series. Convert do_recv_errqueue_timeout into dispatcher do_recv_errqueue and move SO_EE_ORIGIN_TXTIME specific code into a separate helper. This will make the next patch a lot more readable. No functional changes. Signed-off-by: Willem de Bruijn Link: https://patch.msgid.link/20260910171131.2532487-6-willemdebruijn.kernel@gmail.com Signed-off-by: Jakub Kicinski --- .../testing/selftests/drivers/net/so_txtime.c | 66 ++++++++++--------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/tools/testing/selftests/drivers/net/so_txtime.c b/tools/testing/selftests/drivers/net/so_txtime.c index 55a386f3d1b9..f6f3253ed174 100644 --- a/tools/testing/selftests/drivers/net/so_txtime.c +++ b/tools/testing/selftests/drivers/net/so_txtime.c @@ -158,7 +158,40 @@ static void do_recv_verify_empty(int fdr) error(1, 0, "recv: not empty as expected (%d, %d)", ret, errno); } -static int do_recv_errqueue_timeout(int fdt) +static int do_recv_errqueue_txtime(struct sock_extended_err *err, + const char payload_char) +{ + const char *reason = NULL; + int64_t tstamp = 0; + + switch (err->ee_errno) { + case ECANCELED: + if (err->ee_code != SO_EE_CODE_TXTIME_MISSED) + error(1, 0, "errqueue: unknown ECANCELED %u\n", + err->ee_code); + reason = "missed txtime"; + break; + case EINVAL: + if (err->ee_code != SO_EE_CODE_TXTIME_INVALID_PARAM) + error(1, 0, "errqueue: unknown EINVAL %u\n", + err->ee_code); + reason = "invalid txtime"; + break; + default: + error(1, 0, "errqueue: errno %u code %u\n", + err->ee_errno, err->ee_code); + } + + tstamp = ((int64_t)err->ee_data) << 32 | err->ee_info; + tstamp -= (int64_t)glob_tstart; + tstamp /= 1000 * 1000; + fprintf(stderr, "send: pkt %c at %" PRId64 "ms dropped: %s\n", + payload_char, tstamp, reason); + + return 1; +} + +static int do_recv_errqueue(int fdt) { char control[CMSG_SPACE(sizeof(struct sock_extended_err)) + CMSG_SPACE(sizeof(struct sockaddr_in6))] = {0}; @@ -169,7 +202,6 @@ static int do_recv_errqueue_timeout(int fdt) struct msghdr msg = {0}; struct iovec iov = {0}; struct cmsghdr *cm; - int64_t tstamp = 0; iov.iov_base = data; iov.iov_len = sizeof(data); @@ -181,8 +213,6 @@ static int do_recv_errqueue_timeout(int fdt) msg.msg_controllen = sizeof(control); while (1) { - const char *reason = NULL; - ret = recvmsg(fdt, &msg, MSG_ERRQUEUE); if (ret == -1 && errno == EAGAIN) break; @@ -201,33 +231,9 @@ static int do_recv_errqueue_timeout(int fdt) if (err->ee_origin != SO_EE_ORIGIN_TXTIME) error(1, 0, "errqueue: origin 0x%x\n", err->ee_origin); - switch (err->ee_errno) { - case ECANCELED: - if (err->ee_code != SO_EE_CODE_TXTIME_MISSED) - error(1, 0, "errqueue: unknown ECANCELED %u\n", - err->ee_code); - reason = "missed txtime"; - break; - case EINVAL: - if (err->ee_code != SO_EE_CODE_TXTIME_INVALID_PARAM) - error(1, 0, "errqueue: unknown EINVAL %u\n", - err->ee_code); - reason = "invalid txtime"; - break; - default: - error(1, 0, "errqueue: errno %u code %u\n", - err->ee_errno, err->ee_code); - } - - tstamp = ((int64_t) err->ee_data) << 32 | err->ee_info; - tstamp -= (int64_t) glob_tstart; - tstamp /= 1000 * 1000; - fprintf(stderr, "send: pkt %c at %" PRId64 "ms dropped: %s\n", - data[ret - 1], tstamp, reason); - + num_tstamp += do_recv_errqueue_txtime(err, data[ret - 1]); msg.msg_flags = 0; msg.msg_controllen = sizeof(control); - num_tstamp++; } return num_tstamp; @@ -245,7 +251,7 @@ static void recv_errqueue_msgs(int fdt) error(1, errno, "poll"); if (ret && (pfd.revents & POLLERR)) - num_tstamp += do_recv_errqueue_timeout(fdt); + num_tstamp += do_recv_errqueue(fdt); if (num_tstamp == cfg_num_pkt) break; From 4e50c0265f2f96e60dc967539d3b85b90038da26 Mon Sep 17 00:00:00 2001 From: Willem de Bruijn Date: Thu, 10 Sep 2026 13:10:25 -0400 Subject: [PATCH 6/7] selftests: drv-net: in so_txtime tell apart sw from hw pacing Detect software pacing in so_txtime.c using SO_TIMESTAMPING. If '-H' (hw) is passed 1. measure sw tx delay with SOF_TIMESTAMPING_TX_SOFTWARE, and 2. fail if delay exceeds a threshold, indicating pacing This will be used in the next patch in the series. Also extend while condition to account for possible variance. This applies to all tests, not just the new '-H' variants. Also reorder getopt parameters to make them alphabetical. Signed-off-by: Willem de Bruijn Link: https://patch.msgid.link/20260910171131.2532487-7-willemdebruijn.kernel@gmail.com Signed-off-by: Jakub Kicinski --- .../testing/selftests/drivers/net/so_txtime.c | 97 +++++++++++++++---- 1 file changed, 80 insertions(+), 17 deletions(-) diff --git a/tools/testing/selftests/drivers/net/so_txtime.c b/tools/testing/selftests/drivers/net/so_txtime.c index f6f3253ed174..9ebe9f91227c 100644 --- a/tools/testing/selftests/drivers/net/so_txtime.c +++ b/tools/testing/selftests/drivers/net/so_txtime.c @@ -42,8 +42,10 @@ static bool cfg_machine_slow; static uint64_t cfg_start_time_ns; static int cfg_mark; static bool cfg_rx; +static bool cfg_verify_hw_offload; static uint64_t glob_tstart; +static uint64_t glob_tstart_real; static uint64_t tdeliver_max; static int errors; @@ -191,12 +193,32 @@ static int do_recv_errqueue_txtime(struct sock_extended_err *err, return 1; } -static int do_recv_errqueue(int fdt) +static int do_recv_errqueue_timestamping(struct scm_timestamping *tss) +{ + int64_t ts; + + ts = tss->ts[0].tv_sec * 1000ULL * 1000 * 1000; + ts += tss->ts[0].tv_nsec; + ts -= glob_tstart_real; + ts /= 1000; + + if (ts > cfg_variance_us) { + fprintf(stderr, "sw delay %" PRId64 "us exceeds bounds\n", ts); + if (!cfg_machine_slow) + errors++; + } + + return 1; +} + +static int do_recv_errqueue(int fdt, int *num_ts) { char control[CMSG_SPACE(sizeof(struct sock_extended_err)) + + CMSG_SPACE(sizeof(struct scm_timestamping)) + CMSG_SPACE(sizeof(struct sockaddr_in6))] = {0}; char data[sizeof(struct ethhdr) + sizeof(struct ipv6hdr) + sizeof(struct udphdr) + 1]; + struct scm_timestamping *tss; struct sock_extended_err *err; int ret, num_tstamp = 0; struct msghdr msg = {0}; @@ -222,16 +244,29 @@ static int do_recv_errqueue(int fdt) error(1, 0, "errqueue: flags 0x%x\n", msg.msg_flags); cm = CMSG_FIRSTHDR(&msg); + tss = NULL; + + if (cm->cmsg_level == SOL_SOCKET && + cm->cmsg_type == SCM_TIMESTAMPING) { + tss = (void *)CMSG_DATA(cm); + cm = CMSG_NXTHDR(&msg, cm); + if (!cm) + error(1, 0, "timestamp missing ip err\n"); + } + if (cm->cmsg_level != cfg_errq_level || cm->cmsg_type != cfg_errq_type) error(1, 0, "errqueue: type 0x%x.0x%x\n", cm->cmsg_level, cm->cmsg_type); err = (struct sock_extended_err *)CMSG_DATA(cm); - if (err->ee_origin != SO_EE_ORIGIN_TXTIME) + if (err->ee_origin == SO_EE_ORIGIN_TXTIME) + num_tstamp += do_recv_errqueue_txtime(err, data[ret - 1]); + else if (err->ee_origin == SO_EE_ORIGIN_TIMESTAMPING && tss) + *num_ts += do_recv_errqueue_timestamping(tss); + else error(1, 0, "errqueue: origin 0x%x\n", err->ee_origin); - num_tstamp += do_recv_errqueue_txtime(err, data[ret - 1]); msg.msg_flags = 0; msg.msg_controllen = sizeof(control); } @@ -243,7 +278,7 @@ static void recv_errqueue_msgs(int fdt) { struct pollfd pfd = { .fd = fdt, .events = POLLERR }; const int timeout_ms = 10; - int ret, num_tstamp = 0; + int ret, num_tstamp = 0, num_ts = 0; do { ret = poll(&pfd, 1, timeout_ms); @@ -251,12 +286,20 @@ static void recv_errqueue_msgs(int fdt) error(1, errno, "poll"); if (ret && (pfd.revents & POLLERR)) - num_tstamp += do_recv_errqueue(fdt); + num_tstamp += do_recv_errqueue(fdt, &num_ts); - if (num_tstamp == cfg_num_pkt) + if (num_tstamp == cfg_num_pkt || num_ts == cfg_num_pkt) break; - } while (gettime_ns(cfg_clockid) < tdeliver_max); + } while (gettime_ns(cfg_clockid) < + tdeliver_max + (cfg_variance_us * 1000)); + + if (cfg_verify_hw_offload && num_ts != cfg_num_pkt) { + fprintf(stderr, "missing timestamps: expected %d, got %d\n", + cfg_num_pkt, num_ts); + if (!cfg_machine_slow) + errors++; + } } static void start_time_wait(void) @@ -301,6 +344,17 @@ static void setsockopt_txtime(int fd) error(1, 0, "getsockopt txtime: mismatch"); } +static void setsockopt_txtimestamping(int fd) +{ + int val = SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE | + SOF_TIMESTAMPING_OPT_TSONLY; + + if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING, + &val, sizeof(val))) + error(1, errno, "setsockopt timestamping"); +} + static int setup_tx(struct sockaddr *addr, socklen_t alen) { int fd; @@ -314,6 +368,9 @@ static int setup_tx(struct sockaddr *addr, socklen_t alen) setsockopt_txtime(fd); + if (cfg_verify_hw_offload) + setsockopt_txtimestamping(fd); + if (cfg_mark && setsockopt(fd, SOL_SOCKET, SO_MARK, &cfg_mark, sizeof(cfg_mark))) error(1, errno, "setsockopt mark"); @@ -354,6 +411,8 @@ static void do_test_tx(struct sockaddr *addr, socklen_t alen) start_time_wait(); glob_tstart = gettime_ns(cfg_clockid); + glob_tstart_real = gettime_ns(CLOCK_REALTIME); + tdeliver_max = glob_tstart; for (i = 0; i < cfg_num_pkt; i++) do_send_one(fdt, &cfg_buf[i]); @@ -446,10 +505,11 @@ static void usage(const char *progname) " -6 only IPv6\n" " -c monotonic or tai (default)\n" " -D destination IP address (server)\n" - " -S source IP address (client)\n" - " -r run rx mode\n" - " -t start time (UTC nanoseconds)\n" + " -H verify hardware offload (tx)\n" " -m socket mark\n" + " -r run rx mode\n" + " -S source IP address (client)\n" + " -t start time (UTC nanoseconds)\n" "\n", progname); exit(1); @@ -461,7 +521,7 @@ static void parse_opts(int argc, char **argv) int domain = PF_UNSPEC; int c; - while ((c = getopt(argc, argv, "46c:S:D:rt:m:")) != -1) { + while ((c = getopt(argc, argv, "46c:D:Hm:rS:t:")) != -1) { switch (c) { case '4': if (domain != PF_UNSPEC) @@ -488,21 +548,24 @@ static void parse_opts(int argc, char **argv) else error(1, 0, "unknown clock id %s", optarg); break; - case 'S': - saddr = optarg; - break; case 'D': daddr = optarg; break; + case 'H': + cfg_verify_hw_offload = true; + break; + case 'm': + cfg_mark = strtol(optarg, NULL, 0); + break; case 'r': cfg_rx = true; break; + case 'S': + saddr = optarg; + break; case 't': cfg_start_time_ns = strtoll(optarg, NULL, 0); break; - case 'm': - cfg_mark = strtol(optarg, NULL, 0); - break; default: usage(argv[0]); } From ac48ee533902d405eee5997a1fa1fe10924ed27e Mon Sep 17 00:00:00 2001 From: Willem de Bruijn Date: Thu, 10 Sep 2026 13:10:26 -0400 Subject: [PATCH 7/7] selftests: drv-net: extend so_txtime with hw offload Add two pacing hardware offload variants 1. one that uses FQ to safely offload when within bounds. 2. one that uses pfifo_fast and thus forwards all packets. Verify that the packets are paced in hardware with new flag '-H'. Also increase rcvtimeout significantly to reduce flakiness. Especially for the new beyond_hw_horizon test, which is close to the 100ms limit. But update recv_verify_empty to take MSG_DONTWAIT. That last empty check must not delay each testcase by the receive timeout. Hardware pacing offload can complete packets out of order. So the reverse_order test is expected to pass with pfifo_fast too. Do not test ETF, which does not change its dequeue behavior based on pacing_offload. The pfifo_fast beyond_hw_horizon testcase expects a failure because the packet exceeds the hardware horizon and is transmitted immediately, violating receiver arrival bounds. On slow machines (KSFT_MACHINE_SLOW), timing variance errors are suppressed by the receiver, so relax the failure expectation only for this timing-sensitive case while preserving deterministic checks for other tests (such as ETF invalid txtime). Signed-off-by: Willem de Bruijn Link: https://patch.msgid.link/20260910171131.2532487-8-willemdebruijn.kernel@gmail.com Signed-off-by: Jakub Kicinski --- .../testing/selftests/drivers/net/so_txtime.c | 4 +- .../selftests/drivers/net/so_txtime.py | 80 ++++++++++++++++++- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/drivers/net/so_txtime.c b/tools/testing/selftests/drivers/net/so_txtime.c index 9ebe9f91227c..5bb15498da30 100644 --- a/tools/testing/selftests/drivers/net/so_txtime.c +++ b/tools/testing/selftests/drivers/net/so_txtime.c @@ -155,7 +155,7 @@ static void do_recv_verify_empty(int fdr) char rbuf[1]; int ret; - ret = recv(fdr, rbuf, sizeof(rbuf), 0); + ret = recv(fdr, rbuf, sizeof(rbuf), MSG_DONTWAIT); if (ret != -1 || errno != EAGAIN) error(1, 0, "recv: not empty as expected (%d, %d)", ret, errno); } @@ -380,7 +380,7 @@ static int setup_tx(struct sockaddr *addr, socklen_t alen) static int setup_rx(struct sockaddr *addr, socklen_t alen) { - struct timeval tv = { .tv_usec = 100 * 1000 }; + struct timeval tv = { .tv_usec = 600 * 1000 }; int fd; fd = socket(addr->sa_family, SOCK_DGRAM, 0); diff --git a/tools/testing/selftests/drivers/net/so_txtime.py b/tools/testing/selftests/drivers/net/so_txtime.py index a097fae0b335..66a87205e02d 100755 --- a/tools/testing/selftests/drivers/net/so_txtime.py +++ b/tools/testing/selftests/drivers/net/so_txtime.py @@ -12,10 +12,12 @@ import time from lib.py import ksft_exit, ksft_run, ksft_variants from lib.py import KsftNamedVariant, KsftSkipEx from lib.py import NetDrvEpEnv, bkg, cmd, defer, tc -from lib.py import CmdExitFailure +from lib.py import CmdExitFailure, RtnlFamily, NlError +_HW_OFFLOAD_HORIZON_MS = 50 -def test_so_txtime(cfg, clockid, ipver, args_tx, args_rx, expect_success): +def test_so_txtime(cfg, clockid, ipver, args_tx, args_rx, expect_success, + timing_sensitive=False): """Main function. Run so_txtime as sender and receiver.""" slow_machine = os.environ.get('KSFT_MACHINE_SLOW') @@ -33,12 +35,42 @@ def test_so_txtime(cfg, clockid, ipver, args_tx, args_rx, expect_success): expect_fail = not expect_success if slow_machine: expect_success = False + if timing_sensitive: + expect_fail = None with bkg(cmd_rx, host=cfg.remote, fail=expect_success, expect_fail=expect_fail, exit_wait=True): cmd(cmd_tx) +def _dev_setup_pacing_offload(cfg): + """Configure pacing-offload.""" + rtnl = RtnlFamily() + + try: + link = rtnl.getlink({'ifi-index': cfg.ifindex}) + except NlError as e: + raise KsftSkipEx('getlink not supported by device') from e + + if 'pacing-offload' not in link or \ + 'max-pacing-offload-horizon' not in link: + raise KsftSkipEx('pacing offload not supported by device') + + horizon = _HW_OFFLOAD_HORIZON_MS * 1000_000 + if link['max-pacing-offload-horizon'] < horizon: + raise KsftSkipEx('pacing offload max horizon too small') + + cur_offload = link['pacing-offload'] + rtnl.setlink({ + 'ifi-index': cfg.ifindex, + 'pacing-offload': 1, + }) + defer(rtnl.setlink, { + 'ifi-index': cfg.ifindex, + 'pacing-offload': cur_offload + }) + + def _qdisc_setup(ifname, qdisc, optargs=""): """Replace root qdisc. Restore the original after the test. @@ -61,6 +93,7 @@ def _test_variants_fq(): ["one_pkt", "a,10", "a,10"], ["in_order", "a,10,b,20", "a,10,b,20"], ["reverse_order", "a,20,b,10", "b,10,a,20"], + ["beyond_hw_horizon", "a,70", "a,70"], ]: name = f"v{ipver}_{testcase[0]}" yield KsftNamedVariant(name, ipver, testcase[1], testcase[2]) @@ -74,6 +107,41 @@ def test_so_txtime_fq_mono(cfg, ipver, args_tx, args_rx): test_so_txtime(cfg, "mono", ipver, args_tx, args_rx, True) +@ksft_variants(_test_variants_fq()) +def test_so_txtime_fq_mono_hw(cfg, ipver, args_tx, args_rx): + """Run all variants of monotonic fq tests, with offload horizon.""" + cfg.require_ipver(ipver) + cfg.require_nsim(nsim_test=False) + + _dev_setup_pacing_offload(cfg) + try: + _qdisc_setup(cfg.ifname, "fq", f"offload_horizon {_HW_OFFLOAD_HORIZON_MS}ms") + except Exception as e: + raise KsftSkipEx("netdev does not support offload. skipping") from e + + # Expect all tests to use only hw pacing, except beyond_hw_horizon. + # Do not pass -H to that test so that with sw pacing fallback it passes. + hw_only = "-H" if args_tx != "a,70" else "" + test_so_txtime(cfg, "mono", ipver, f"{hw_only} {args_tx}", args_rx, True) + + +@ksft_variants(_test_variants_fq()) +def test_so_txtime_pfifofast_mono_hw(cfg, ipver, args_tx, args_rx): + """Run all variants of monotonic tests, without fq pacing sw backup.""" + cfg.require_ipver(ipver) + cfg.require_nsim(nsim_test=False) + + _dev_setup_pacing_offload(cfg) + _qdisc_setup(cfg.ifname, "pfifo_fast") + + # Expect all tests to pass, except beyond_hw_horizon without sw fallback. + # It will send immediately, failing the receiver arrival bounds check. + expect_pass = args_tx != "a,70" + timing_sensitive = not expect_pass + test_so_txtime(cfg, "mono", ipver, f"-H {args_tx}", args_rx, expect_pass, + timing_sensitive=timing_sensitive) + + @ksft_variants(_test_variants_fq()) def test_so_txtime_fq_tai(cfg, ipver, args_tx, args_rx): """Run all variants of fq tests, but pass CLOCK_TAI to test conversion.""" @@ -123,7 +191,13 @@ def main() -> None: """Boilerplate ksft main.""" with NetDrvEpEnv(__file__) as cfg: ksft_run( - [test_so_txtime_fq_mono, test_so_txtime_fq_tai, test_so_txtime_etf], + [ + test_so_txtime_fq_mono, + test_so_txtime_fq_mono_hw, + test_so_txtime_pfifofast_mono_hw, + test_so_txtime_fq_tai, + test_so_txtime_etf, + ], args=(cfg,), ) ksft_exit()