diff --git a/include/uapi/linux/pkt_sched.h b/include/uapi/linux/pkt_sched.h index 66e8072f44df..490efd288526 100644 --- a/include/uapi/linux/pkt_sched.h +++ b/include/uapi/linux/pkt_sched.h @@ -569,6 +569,16 @@ struct tc_netem_gemodel { #define NETEM_DIST_SCALE 8192 #define NETEM_DIST_MAX 16384 +struct tc_netem_xstats { + __u64 delayed; /* packets delayed */ + __u64 dropped; /* packets dropped by loss model */ + __u64 corrupted; /* packets with bit errors injected */ + __u64 duplicated; /* duplicate packets generated */ + __u64 reordered; /* packets sent out of order */ + __u64 ecn_marked; /* packets ECN CE-marked (not dropped)*/ + __u64 allocation_errors; +}; + /* DRR */ enum { diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index ddbfea9dd32a..6cd1838e09e7 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -27,8 +27,6 @@ #include #include -#define VERSION "1.3" - /* Network Emulation Queuing algorithm. ==================================== @@ -71,89 +69,101 @@ struct disttable { s16 table[] __counted_by(size); }; +/* Loss models */ +enum { + CLG_RANDOM, + CLG_4_STATES, + CLG_GILB_ELL, +}; + +/* States in GE model */ +enum { + GOOD_STATE = 1, + BAD_STATE, +}; + +/* States in 4 state model */ +enum { + TX_IN_GAP_PERIOD = 1, + TX_IN_BURST_PERIOD, + LOST_IN_GAP_PERIOD, + LOST_IN_BURST_PERIOD, +}; + struct netem_sched_data { - /* internal t(ime)fifo qdisc uses t_root and sch->limit */ - struct rb_root t_root; - - /* a linear queue; reduces rbtree rebalancing when jitter is low */ - struct sk_buff *t_head; - struct sk_buff *t_tail; - - u32 t_len; - - /* optional qdisc for classful handling (NULL at netem init) */ - struct Qdisc *qdisc; - - struct qdisc_watchdog watchdog; - - s64 latency; - s64 jitter; - - u32 loss; - u32 ecn; - u32 limit; - u32 counter; - u32 gap; - u32 duplicate; - u32 reorder; - u32 corrupt; - u64 rate; - s32 packet_overhead; - u32 cell_size; - struct reciprocal_value cell_size_reciprocal; - s32 cell_overhead; + /* Cacheline 0: tfifo state and per-packet enqueue/dequeue scalars. */ + struct rb_root t_root; + struct sk_buff *t_head; + struct sk_buff *t_tail; + u32 t_len; + u32 counter; + s64 latency; + s64 jitter; + u64 rate; + u32 gap; + u32 loss; + /* Cacheline 1: zero-check scalars and correlation states. */ + u32 duplicate; + u32 reorder; + u32 corrupt; + u32 ecn; struct crndstate { u32 last; u32 rho; } delay_cor, loss_cor, dup_cor, reorder_cor, corrupt_cor; + u8 loss_model; - struct prng { + /* Cacheline 2: PRNG, distribution tables, slot dequeue state etc. */ + struct prng { u64 seed; struct rnd_state prng_state; } prng; + struct disttable *delay_dist; + struct slotstate { + u64 slot_next; + s32 packets_left; + s32 bytes_left; + } slot; + struct disttable *slot_dist; + struct Qdisc *qdisc; - struct disttable *delay_dist; - - enum { - CLG_RANDOM, - CLG_4_STATES, - CLG_GILB_ELL, - } loss_model; - - enum { - TX_IN_GAP_PERIOD = 1, - TX_IN_BURST_PERIOD, - LOST_IN_GAP_PERIOD, - LOST_IN_BURST_PERIOD, - } _4_state_model; - - enum { - GOOD_STATE = 1, - BAD_STATE, - } GE_state_model; + /* + * Warm: rate-shaping parameters (only read when rate != 0) and + * configuration-only fields. The fast path reads sch->limit, not + * q->limit. + */ + s32 packet_overhead; + u32 cell_size; + struct reciprocal_value cell_size_reciprocal; + s32 cell_overhead; + u32 limit; /* Correlated Loss Generation models */ struct clgstate { - /* state of the Markov chain */ - u8 state; - /* 4-states and Gilbert-Elliot models */ u32 a1; /* p13 for 4-states or p for GE */ u32 a2; /* p31 for 4-states or r for GE */ u32 a3; /* p32 for 4-states or h for GE */ u32 a4; /* p14 for 4-states or 1-k for GE */ u32 a5; /* p23 used only in 4-states */ + + /* state of the Markov chain */ + u8 state; } clg; - struct tc_netem_slot slot_config; - struct slotstate { - u64 slot_next; - s32 packets_left; - s32 bytes_left; - } slot; + /* Impairment counters */ + u64 delayed; + u64 dropped; + u64 corrupted; + u64 duplicated; + u64 ecn_marked; + u64 reordered; + u64 allocation_errors; - struct disttable *slot_dist; + /* Cold tail: slot reschedule config and the watchdog timer. */ + struct tc_netem_slot slot_config; + struct qdisc_watchdog watchdog; }; /* Time stamp put into socket buffer control block @@ -461,16 +471,21 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, skb->prev = NULL; /* Random duplication */ - if (q->duplicate && q->duplicate >= get_crandom(&q->dup_cor, &q->prng)) + if (q->duplicate && q->duplicate >= get_crandom(&q->dup_cor, &q->prng)) { ++count; + WRITE_ONCE(q->duplicated, q->duplicated + 1); + } /* Drop packet? */ if (loss_event(q)) { - if (q->ecn && INET_ECN_set_ce(skb)) - qdisc_qstats_drop(sch); /* mark packet */ - else + if (q->ecn && INET_ECN_set_ce(skb)) { + WRITE_ONCE(q->ecn_marked, q->ecn_marked + 1); + } else { + WRITE_ONCE(q->dropped, q->dropped + 1); --count; + } } + if (count == 0) { qdisc_qstats_drop(sch); __qdisc_drop(skb, to_free); @@ -487,8 +502,11 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, * If we need to duplicate packet, then clone it before * original is modified. */ - if (count > 1) + if (count > 1) { skb2 = skb_clone(skb, GFP_ATOMIC); + if (!skb2) + WRITE_ONCE(q->allocation_errors, q->allocation_errors + 1); + } /* * Randomized packet corruption. @@ -499,8 +517,10 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (q->corrupt && q->corrupt >= get_crandom(&q->corrupt_cor, &q->prng)) { if (skb_is_gso(skb)) { skb = netem_segment(skb, sch, to_free); - if (!skb) + if (!skb) { + WRITE_ONCE(q->allocation_errors, q->allocation_errors + 1); goto finish_segs; + } segs = skb->next; skb_mark_not_on_list(skb); @@ -509,19 +529,23 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, skb = skb_unshare(skb, GFP_ATOMIC); if (unlikely(!skb)) { + WRITE_ONCE(q->allocation_errors, q->allocation_errors + 1); qdisc_qstats_drop(sch); goto finish_segs; } - if (skb->ip_summed == CHECKSUM_PARTIAL && - skb_checksum_help(skb)) { + if (skb_linearize(skb) || + (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb))) { + WRITE_ONCE(q->allocation_errors, q->allocation_errors + 1); qdisc_drop(skb, sch, to_free); skb = NULL; goto finish_segs; } - if (skb_headlen(skb)) - skb->data[get_random_u32_below(skb_headlen(skb))] ^= - 1 << get_random_u32_below(8); + if (skb->len) { + u32 offset = get_random_u32_below(skb->len); + skb->data[offset] ^= 1 << get_random_u32_below(8); + WRITE_ONCE(q->corrupted, q->corrupted + 1); + } } if (unlikely(sch->q.qlen >= sch->limit)) { @@ -602,12 +626,16 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, cb->time_to_send = now + delay; ++q->counter; + if (delay) + WRITE_ONCE(q->delayed, q->delayed + 1); + tfifo_enqueue(skb, sch); } else { /* * Do re-ordering by putting one out of N packets at the front * of the queue. */ + WRITE_ONCE(q->reordered, q->reordered + 1); cb->time_to_send = ktime_get_ns(); q->counter = 0; @@ -920,7 +948,8 @@ static void get_rate(struct netem_sched_data *q, const struct nlattr *attr) q->cell_size_reciprocal = (struct reciprocal_value) { 0 }; } -static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr) +static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr, + struct netlink_ext_ack *extack) { const struct nlattr *la; int rem; @@ -933,7 +962,8 @@ static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr) const struct tc_netem_gimodel *gi = nla_data(la); if (nla_len(la) < sizeof(struct tc_netem_gimodel)) { - pr_info("netem: incorrect gi model size\n"); + NL_SET_ERR_MSG_ATTR(extack, la, + "netem: incorrect gi model size"); return -EINVAL; } @@ -952,7 +982,8 @@ static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr) const struct tc_netem_gemodel *ge = nla_data(la); if (nla_len(la) < sizeof(struct tc_netem_gemodel)) { - pr_info("netem: incorrect ge model size\n"); + NL_SET_ERR_MSG_ATTR(extack, la, + "netem: incorrect ge model size"); return -EINVAL; } @@ -966,7 +997,8 @@ static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr) } default: - pr_info("netem: unknown loss type %u\n", type); + NL_SET_ERR_MSG_ATTR_FMT(extack, la, + "netem: unknown loss type %u", type); return -EINVAL; } } @@ -989,19 +1021,21 @@ static const struct nla_policy netem_policy[TCA_NETEM_MAX + 1] = { }; static int parse_attr(struct nlattr *tb[], int maxtype, struct nlattr *nla, - const struct nla_policy *policy, int len) + const struct nla_policy *policy, int len, + struct netlink_ext_ack *extack) { int nested_len = nla_len(nla) - NLA_ALIGN(len); if (nested_len < 0) { - pr_info("netem: invalid attributes len %d\n", nested_len); + NL_SET_ERR_MSG_FMT(extack, "netem: invalid attributes len %d < %d", + nla_len(nla), NLA_ALIGN(len)); return -EINVAL; } if (nested_len >= nla_attr_size(0)) return nla_parse_deprecated(tb, maxtype, nla_data(nla) + NLA_ALIGN(len), - nested_len, policy, NULL); + nested_len, policy, extack); memset(tb, 0, sizeof(struct nlattr *) * (maxtype + 1)); return 0; @@ -1056,7 +1090,7 @@ static int netem_change(struct Qdisc *sch, struct nlattr *opt, int ret; qopt = nla_data(opt); - ret = parse_attr(tb, TCA_NETEM_MAX, opt, netem_policy, sizeof(*qopt)); + ret = parse_attr(tb, TCA_NETEM_MAX, opt, netem_policy, sizeof(*qopt), extack); if (ret < 0) return ret; @@ -1096,7 +1130,7 @@ static int netem_change(struct Qdisc *sch, struct nlattr *opt, old_loss_model = q->loss_model; if (tb[TCA_NETEM_LOSS]) { - ret = get_loss_clg(q, tb[TCA_NETEM_LOSS]); + ret = get_loss_clg(q, tb[TCA_NETEM_LOSS], extack); if (ret) { q->loss_model = old_loss_model; q->clg = old_clg; @@ -1180,7 +1214,6 @@ static int netem_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { struct netem_sched_data *q = qdisc_priv(sch); - int ret; qdisc_watchdog_init(&q->watchdog, sch); @@ -1191,10 +1224,7 @@ static int netem_init(struct Qdisc *sch, struct nlattr *opt, q->prng.seed = get_random_u64(); prandom_seed_state(&q->prng.prng_state, q->prng.seed); - ret = netem_change(sch, opt, extack); - if (ret) - pr_info("netem: change failed\n"); - return ret; + return netem_change(sch, opt, extack); } static void netem_destroy(struct Qdisc *sch) @@ -1344,6 +1374,22 @@ nla_put_failure: return -1; } +static int netem_dump_stats(struct Qdisc *sch, struct gnet_dump *d) +{ + struct netem_sched_data *q = qdisc_priv(sch); + struct tc_netem_xstats st = { + .delayed = READ_ONCE(q->delayed), + .dropped = READ_ONCE(q->dropped), + .corrupted = READ_ONCE(q->corrupted), + .duplicated = READ_ONCE(q->duplicated), + .reordered = READ_ONCE(q->reordered), + .ecn_marked = READ_ONCE(q->ecn_marked), + .allocation_errors = READ_ONCE(q->allocation_errors), + }; + + return gnet_stats_copy_app(d, &st, sizeof(st)); +} + static int netem_dump_class(struct Qdisc *sch, unsigned long cl, struct sk_buff *skb, struct tcmsg *tcm) { @@ -1406,14 +1452,13 @@ static struct Qdisc_ops netem_qdisc_ops __read_mostly = { .destroy = netem_destroy, .change = netem_change, .dump = netem_dump, + .dump_stats = netem_dump_stats, .owner = THIS_MODULE, }; MODULE_ALIAS_NET_SCH("netem"); - static int __init netem_module_init(void) { - pr_info("netem: version " VERSION "\n"); return register_qdisc(&netem_qdisc_ops); } static void __exit netem_module_exit(void)