diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h index 470bc23c844c..f214023095ee 100644 --- a/drivers/net/ethernet/intel/idpf/idpf.h +++ b/drivers/net/ethernet/intel/idpf/idpf.h @@ -604,6 +604,16 @@ struct idpf_vport_config { DECLARE_BITMAP(flags, IDPF_VPORT_CONFIG_FLAGS_NBITS); }; +/** + * struct idpf_edt_caps_ilog2 - Host parsed EDT capabilities. + * @time_horizon_ns: Total time window in nanoseconds. + * @tstamp_granularity_pow2: Log2 of timestamp granularity in nanoseconds. + */ +struct idpf_edt_caps_ilog2 { + u32 time_horizon_ns; + u8 tstamp_granularity_pow2; +}; + #define idpf_for_each_vport(adapter, iter) \ for (struct idpf_vport **__##iter = &(adapter)->vports[0], \ *iter = (adapter)->max_vports ? *__##iter : NULL; \ @@ -657,6 +667,7 @@ struct idpf_vport_config { * @stats_task: Periodic statistics retrieval task * @stats_wq: Workqueue for statistics task * @caps: Negotiated capabilities with device + * @edt_caps: Negotiated EDT capabilities with device * @dev_ops: See idpf_dev_ops * @cdev_info: IDC core device info pointer * @num_vfs: Number of allocated VFs through sysfs. PF does not directly talk @@ -720,6 +731,7 @@ struct idpf_adapter { struct delayed_work stats_task; struct workqueue_struct *stats_wq; struct virtchnl2_get_capabilities caps; + struct idpf_edt_caps_ilog2 edt_caps; struct idpf_dev_ops dev_ops; struct iidc_rdma_core_dev_info *cdev_info; diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index 827c795afcb6..cd1b173e3b27 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -817,6 +817,21 @@ static void idpf_attach_and_open(struct idpf_adapter *adapter) } } +static void idpf_vport_set_pacing_offload(struct idpf_vport *vport, + struct net_device *netdev) +{ + struct idpf_adapter *adapter = vport->adapter; + u32 max_horizon = 0; + + if (adapter->edt_caps.time_horizon_ns && + idpf_is_queue_model_split(vport->dflt_qv_rsrc.txq_model) && + !idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, + VIRTCHNL2_CAP_SPLITQ_QSCHED)) + max_horizon = adapter->edt_caps.time_horizon_ns; + + WRITE_ONCE(netdev->max_pacing_offload_horizon, max_horizon); +} + /** * idpf_cfg_netdev - Allocate, configure and register a netdev * @vport: main vport structure @@ -850,6 +865,8 @@ static int idpf_cfg_netdev(struct idpf_vport *vport) np->max_tx_hdr_size = idpf_get_max_tx_hdr_size(adapter); vport->netdev = netdev; + idpf_vport_set_pacing_offload(vport, netdev); + return idpf_init_mac_addr(vport, netdev); } @@ -890,6 +907,8 @@ static int idpf_cfg_netdev(struct idpf_vport *vport) netdev->min_mtu = ETH_MIN_MTU; netdev->max_mtu = vport->max_mtu; + idpf_vport_set_pacing_offload(vport, netdev); + dflt_features = NETIF_F_SG | NETIF_F_HIGHDMA; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 4311ffa30bb1..215a62161a04 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -1742,6 +1742,7 @@ static int idpf_txq_group_alloc(struct idpf_vport *vport, q->desc_count = rsrc->txq_desc_count; q->tx_max_bufs = idpf_get_max_tx_bufs(adapter); q->tx_min_pkt_len = idpf_get_min_tx_pkt_len(adapter); + q->ts_gran_pow2 = adapter->edt_caps.tstamp_granularity_pow2; q->netdev = vport->netdev; q->txq_grp = tx_qgrp; q->rel_q_id = j; @@ -2408,7 +2409,12 @@ void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc, struct idpf_tx_splitq_params *params, u16 td_cmd, u16 size) { - *(__le32 *)&desc->flow.qw1.cmd_dtype = cpu_to_le32((u8)(params->dtype | td_cmd)); + desc->flow.qw1.cmd_dtype = (u8)(params->dtype | td_cmd); + + desc->flow.qw1.ts[0] = params->offload.desc_ts[0]; + desc->flow.qw1.ts[1] = params->offload.desc_ts[1]; + desc->flow.qw1.ts[2] = params->offload.desc_ts[2]; + desc->flow.qw1.rxr_bufsize = cpu_to_le16((u16)size); desc->flow.qw1.compl_tag = cpu_to_le16(params->compl_tag); } @@ -3011,6 +3017,63 @@ static bool idpf_tx_splitq_need_re(struct idpf_tx_queue *tx_q) return gap >= IDPF_TX_SPLITQ_RE_MIN_GAP; } +static void idpf_tx_splitq_set_txtime(const struct sk_buff *skb, + const struct idpf_tx_queue *tx_q, + struct idpf_tx_splitq_params *tx_params) +{ + const int offload_slack_ns = 400; + u64 ts, now, horizon; + + if (!tx_q->netdev->pacing_offload) + return; + + horizon = READ_ONCE(tx_q->netdev->max_pacing_offload_horizon); + if (!horizon) + return; + + switch (skb->tstamp_type) { + case SKB_CLOCK_REALTIME: + ts = ktime_to_ns(ktime_add(skb->tstamp, + ktime_mono_to_any(0, TK_OFFS_TAI) - + ktime_mono_to_any(0, TK_OFFS_REAL))); + break; + case SKB_CLOCK_MONOTONIC: + ts = ktime_to_ns(ktime_mono_to_any(skb->tstamp, TK_OFFS_TAI)); + break; + case SKB_CLOCK_TAI: + ts = ktime_to_ns(skb->tstamp); + break; + default: + WARN_ON_ONCE(1); + return; + } + + now = ktime_get_clocktai_ns(); + if (ts < now + offload_slack_ns) + return; + + /* beyond offload horizon? set overflow bit only */ + if (ts > now + horizon) { + tx_params->offload.desc_ts[2] = + IDPF_TXD_FLOW_SCH_HORIZON_OVERFLOW_M; + return; + } + + ts >>= tx_q->ts_gran_pow2; + + /* 0 is valid 23b timestamp, but also means field unset. + * Increase by one to avoid this case + */ + if ((ts & 0x7fffff) == 0) { + tx_params->offload.desc_ts[0] = 1; + return; + } + + tx_params->offload.desc_ts[0] = ts & 0xff; + tx_params->offload.desc_ts[1] = (ts >> 8) & 0xff; + tx_params->offload.desc_ts[2] = ((ts >> 16) & 0x7f); +} + /** * idpf_tx_splitq_frame - Sends buffer on Tx ring using flex descriptors * @skb: send buffer @@ -3097,6 +3160,10 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_FLOW_SCHE; tx_params.eop_cmd = IDPF_TXD_FLEX_FLOW_CMD_EOP; + + if (skb->tstamp) + idpf_tx_splitq_set_txtime(skb, tx_q, &tx_params); + /* Set the RE bit periodically to "clean" the descriptor ring */ if (idpf_tx_splitq_need_re(tx_q)) { tx_params.eop_cmd |= IDPF_TXD_FLEX_FLOW_CMD_RE; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 93547597efd2..fe0c913f9bb9 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -161,6 +161,7 @@ union idpf_tx_flex_desc { * @tso_segs: Number of segments to be sent * @tso_hdr_len: Length of headers to be duplicated * @td_cmd: Command field to be inserted into descriptor + * @desc_ts: Flow scheduling offload timestamp */ struct idpf_tx_offload_params { u32 tx_flags; @@ -174,6 +175,7 @@ struct idpf_tx_offload_params { u16 tso_hdr_len; u16 td_cmd; + u8 desc_ts[3]; }; /** @@ -608,6 +610,7 @@ libeth_cacheline_set_assert(struct idpf_rx_queue, * hot path TX pointers stored in vport. Used in both singleq/splitq. * @desc_count: Number of descriptors * @tx_min_pkt_len: Min supported packet length + * @ts_gran_pow2: Txtime timestamp granularity in nanoseconds (log2). * @thresh: XDP queue cleaning threshold * @netdev: &net_device corresponding to this queue * @next_to_use: Next descriptor to use @@ -666,7 +669,10 @@ struct idpf_tx_queue { u16 desc_count; union { - u16 tx_min_pkt_len; + struct { + u16 tx_min_pkt_len; + u8 ts_gran_pow2; + }; u32 thresh; }; diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c index 1caf52706973..928de868fa62 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c @@ -3,6 +3,7 @@ #include #include +#include #include #include "idpf.h" @@ -553,7 +554,8 @@ static int idpf_send_get_caps_msg(struct idpf_adapter *adapter) VIRTCHNL2_CAP_SPLITQ_QSCHED | VIRTCHNL2_CAP_PROMISC | VIRTCHNL2_CAP_LOOPBACK | - VIRTCHNL2_CAP_PTP); + VIRTCHNL2_CAP_PTP | + VIRTCHNL2_CAP_EDT); err = idpf_send_mb_msg_stack(adapter, &xn_params, &caps); if (err) @@ -573,6 +575,54 @@ free_rx_buf: return err; } +/** + * idpf_send_get_edt_caps_msg - Send virtchnl get EDT caps msg + * @adapter: Driver specific private struct + * + * Return: 0 on success or error code on failure. + */ +static int idpf_send_get_edt_caps_msg(struct idpf_adapter *adapter) +{ + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_GET_EDT_CAPS, + }; + struct virtchnl2_edt_caps caps = {}; + u64 gran_ns, horizon_ns; + int err; + + err = idpf_send_mb_msg_stack(adapter, &xn_params, &caps); + if (err) + return err; + + if (xn_params.recv_mem.iov_len < sizeof(caps)) { + err = -EIO; + goto free_rx_buf; + } + + memcpy(&caps, xn_params.recv_mem.iov_base, sizeof(caps)); + horizon_ns = le64_to_cpu(caps.time_horizon_ns); + gran_ns = le64_to_cpu(caps.tstamp_granularity_ns); + if (horizon_ns > U32_MAX) { + dev_warn(&adapter->pdev->dev, "EDT horizon exceeds U32\n"); + err = -EINVAL; + goto free_rx_buf; + } + if (!gran_ns || !is_power_of_2(gran_ns)) { + dev_warn(&adapter->pdev->dev, "Invalid EDT granularity\n"); + err = -EINVAL; + goto free_rx_buf; + } + + adapter->edt_caps.time_horizon_ns = horizon_ns; + adapter->edt_caps.tstamp_granularity_pow2 = ilog2(gran_ns); + +free_rx_buf: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + + return err; +} + /** * idpf_mmio_region_non_static - Check if region is not static * @mmio_info: PCI resources info @@ -3085,6 +3135,14 @@ restart: } } + memset(&adapter->edt_caps, 0, sizeof(adapter->edt_caps)); + if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_EDT)) { + err = idpf_send_get_edt_caps_msg(adapter); + if (err) + dev_err(&adapter->pdev->dev, + "EDT init failed, err=%d\n", err); + } + pci_sriov_set_totalvfs(adapter->pdev, idpf_get_max_vfs(adapter)); num_max_vports = idpf_get_max_vports(adapter); adapter->vports = kzalloc_objs(*adapter->vports, num_max_vports);