diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h index d2813eb795be..be3a6617adf4 100644 --- a/include/net/ip_vs.h +++ b/include/net/ip_vs.h @@ -36,6 +36,12 @@ #define IP_VS_HDR_INVERSE 1 #define IP_VS_HDR_ICMP 2 +/* Destination Server Flags */ +#define IP_VS_DEST_F_OVERLOAD 0x0002 /* server is overloaded */ + +/* Destination Server Config Flags */ +#define IP_VS_DEST_CF_AVAILABLE 0x0001 /* server is available */ + /* conn_tab limits (as per Kconfig) */ #define IP_VS_CONN_TAB_MIN_BITS 8 #if BITS_PER_LONG > 32 @@ -976,6 +982,7 @@ struct ip_vs_dest { volatile unsigned int flags; /* dest status flags */ atomic_t conn_flags; /* flags to copy to conn */ atomic_t weight; /* server weight */ + unsigned long cflags; /* config flags */ atomic_t last_weight; /* server latest weight */ __u16 tun_type; /* tunnel type */ __be16 tun_port; /* tunnel port */ @@ -987,10 +994,11 @@ struct ip_vs_dest { /* connection counters and thresholds */ atomic_t activeconns; /* active connections */ - atomic_t inactconns; /* inactive connections */ + atomic_t totalconns; /* total connections */ atomic_t persistconns; /* persistent connections */ __u32 u_threshold; /* upper threshold */ __u32 l_threshold; /* lower threshold */ + __u32 l_threshold_val;/* used lower threshold */ /* for destination cache */ spinlock_t dst_lock; /* lock of dst_cache */ @@ -1907,6 +1915,8 @@ static inline void ip_vs_dest_put_and_free(struct ip_vs_dest *dest) kfree(dest); } +void ip_vs_dest_update_overload(struct ip_vs_dest *dest, int mode); + /* IPVS sync daemon data and function prototypes * (from ip_vs_sync.c) */ @@ -2058,7 +2068,7 @@ static inline bool ip_vs_conn_use_hash2(struct ip_vs_conn *cp) !(cp->flags & IP_VS_CONN_F_TEMPLATE); } -void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, +bool ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, struct ip_vs_conn *cp, int dir, unsigned int toff, bool has_ports, struct ip_vs_iphdr *ciph); @@ -2220,14 +2230,21 @@ void ip_vs_unregister_hooks(struct netns_ipvs *ipvs, unsigned int af); static inline int ip_vs_dest_conn_overhead(struct ip_vs_dest *dest) { - /* We think the overhead of processing active connections is 256 + /* We think the overhead of processing active connections is 257 * times higher than that of inactive connections in average. (This - * 256 times might not be accurate, we will change it later) We + * 257 times might not be accurate, we will change it later) We * use the following formula to estimate the overhead now: - * dest->activeconns*256 + dest->inactconns + * dest->activeconns*256 + dest->totalconns */ return (atomic_read(&dest->activeconns) << 8) + - atomic_read(&dest->inactconns); + atomic_read(&dest->totalconns); +} + +static inline int +ip_vs_dest_inactconns(const struct ip_vs_dest *dest) +{ + return max(atomic_read(&dest->totalconns) - + atomic_read(&dest->activeconns), 0); } #ifdef CONFIG_IP_VS_PROTO_TCP diff --git a/include/uapi/linux/ip_vs.h b/include/uapi/linux/ip_vs.h index 1ed234e7f251..2c37c6ac7525 100644 --- a/include/uapi/linux/ip_vs.h +++ b/include/uapi/linux/ip_vs.h @@ -28,12 +28,6 @@ #define IP_VS_SVC_F_SCHED_SH_FALLBACK IP_VS_SVC_F_SCHED1 /* SH fallback */ #define IP_VS_SVC_F_SCHED_SH_PORT IP_VS_SVC_F_SCHED2 /* SH use port */ -/* - * Destination Server Flags - */ -#define IP_VS_DEST_F_AVAILABLE 0x0001 /* server is available */ -#define IP_VS_DEST_F_OVERLOAD 0x0002 /* server is overloaded */ - /* * IPVS sync daemon states */ diff --git a/net/bridge/netfilter/nf_conntrack_bridge.c b/net/bridge/netfilter/nf_conntrack_bridge.c index 58a33d0380b0..7ecb8a26bfa3 100644 --- a/net/bridge/netfilter/nf_conntrack_bridge.c +++ b/net/bridge/netfilter/nf_conntrack_bridge.c @@ -280,6 +280,7 @@ static unsigned int nf_ct_bridge_pre(void *priv, struct sk_buff *skb, ret = nf_ct_br_defrag6(skb, &bridge_state); break; default: + nf_reset_ct(skb); nf_ct_set(skb, NULL, IP_CT_UNTRACKED); return NF_ACCEPT; } diff --git a/net/netfilter/ipset/ip_set_bitmap_gen.h b/net/netfilter/ipset/ip_set_bitmap_gen.h index 226fdf17b683..d6a7e6604542 100644 --- a/net/netfilter/ipset/ip_set_bitmap_gen.h +++ b/net/netfilter/ipset/ip_set_bitmap_gen.h @@ -77,7 +77,7 @@ mtype_flush(struct ip_set *set) mtype_ext_cleanup(set); bitmap_zero(map->members, map->elements); set->elements = 0; - atomic64_set(&set->ext_size, 0); + DEBUG_NET_WARN_ON_ONCE(atomic64_read(&set->ext_size) > 0); } /* Calculate the actual memory size of the set data */ diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c index 543851a923d0..0a86a170ba90 100644 --- a/net/netfilter/ipset/ip_set_core.c +++ b/net/netfilter/ipset/ip_set_core.c @@ -679,12 +679,19 @@ __ip_set_get(struct ip_set *set) write_unlock_bh(&ip_set_ref_lock); } +static void +__ip_set_put_locked(struct ip_set *set) +{ + lockdep_assert_held(&ip_set_ref_lock); + BUG_ON(set->ref == 0); + set->ref--; +} + static void __ip_set_put(struct ip_set *set) { write_lock_bh(&ip_set_ref_lock); - BUG_ON(set->ref == 0); - set->ref--; + __ip_set_put_locked(set); write_unlock_bh(&ip_set_ref_lock); } @@ -855,11 +862,11 @@ __ip_set_put_byindex(struct ip_set_net *inst, ip_set_id_t index) { struct ip_set *set; - rcu_read_lock(); - set = rcu_dereference(inst->ip_set_list)[index]; + write_lock_bh(&ip_set_ref_lock); + set = ip_set(inst, index); if (set) - __ip_set_put(set); - rcu_read_unlock(); + __ip_set_put_locked(set); + write_unlock_bh(&ip_set_ref_lock); } void diff --git a/net/netfilter/ipset/ip_set_list_set.c b/net/netfilter/ipset/ip_set_list_set.c index ca3ef9479e83..f070088742d6 100644 --- a/net/netfilter/ipset/ip_set_list_set.c +++ b/net/netfilter/ipset/ip_set_list_set.c @@ -301,9 +301,12 @@ list_set_uadd(struct ip_set *set, void *value, const struct ip_set_ext *ext, e->set = set; INIT_LIST_HEAD(&e->list); list_set_init_extensions(set, ext, e); - if (n) + if (n) { list_set_replace(set, e, n); - else if (next) + return 0; + } + + if (next) list_add_tail_rcu(&e->list, &next->list); else if (prev) list_add_rcu(&e->list, &prev->list); @@ -420,8 +423,7 @@ list_set_flush(struct ip_set *set) list_for_each_entry_safe(e, n, &map->members, list) list_set_del(set, e); - set->elements = 0; - atomic64_set(&set->ext_size, 0); + DEBUG_NET_WARN_ON_ONCE(set->elements > 0); } static void diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index 36c5cba03f5b..6fa3e1dc534c 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -1081,12 +1081,6 @@ static inline void ip_vs_bind_xmit_v6(struct ip_vs_conn *cp) #endif -static inline int ip_vs_dest_totalconns(struct ip_vs_dest *dest) -{ - return atomic_read(&dest->activeconns) - + atomic_read(&dest->inactconns); -} - /* * Bind a connection entry with a virtual service destination * Called just after a new connection entry is created. @@ -1147,23 +1141,22 @@ ip_vs_bind_dest(struct ip_vs_conn *cp, struct ip_vs_dest *dest) /* Update the connection counters */ if (!(flags & IP_VS_CONN_F_TEMPLATE)) { + int tc; + /* It is a normal connection, so modify the counters * according to the flags, later the protocol can * update them on state change */ if (!(flags & IP_VS_CONN_F_INACTIVE)) atomic_inc(&dest->activeconns); - else - atomic_inc(&dest->inactconns); + tc = atomic_inc_return(&dest->totalconns); + if (tc == READ_ONCE(dest->u_threshold)) + ip_vs_dest_update_overload(dest, 1); } else { /* It is a persistent connection/template, so increase the persistent connection counter */ atomic_inc(&dest->persistconns); } - - if (dest->u_threshold != 0 && - ip_vs_dest_totalconns(dest) >= dest->u_threshold) - dest->flags |= IP_VS_DEST_F_OVERLOAD; } @@ -1244,30 +1237,20 @@ static inline void ip_vs_unbind_dest(struct ip_vs_conn *cp) /* Update the connection counters */ if (!(cp->flags & IP_VS_CONN_F_TEMPLATE)) { - /* It is a normal connection, so decrease the inactconns - or activeconns counter */ - if (cp->flags & IP_VS_CONN_F_INACTIVE) { - atomic_dec(&dest->inactconns); - } else { + int tc; + + /* It is a normal connection, so decrease the counters */ + if (!(cp->flags & IP_VS_CONN_F_INACTIVE)) atomic_dec(&dest->activeconns); - } + tc = atomic_fetch_dec(&dest->totalconns); + if (tc == READ_ONCE(dest->l_threshold_val)) + ip_vs_dest_update_overload(dest, -1); } else { /* It is a persistent connection/template, so decrease the persistent connection counter */ atomic_dec(&dest->persistconns); } - if (dest->l_threshold != 0) { - if (ip_vs_dest_totalconns(dest) < dest->l_threshold) - dest->flags &= ~IP_VS_DEST_F_OVERLOAD; - } else if (dest->u_threshold != 0) { - if (ip_vs_dest_totalconns(dest) * 4 < dest->u_threshold * 3) - dest->flags &= ~IP_VS_DEST_F_OVERLOAD; - } else { - if (dest->flags & IP_VS_DEST_F_OVERLOAD) - dest->flags &= ~IP_VS_DEST_F_OVERLOAD; - } - ip_vs_dest_put(dest); } @@ -1296,7 +1279,7 @@ int ip_vs_check_template(struct ip_vs_conn *ct, struct ip_vs_dest *cdest) * Checking the dest server status. */ if ((dest == NULL) || - !(dest->flags & IP_VS_DEST_F_AVAILABLE) || + !(dest->cflags & IP_VS_DEST_CF_AVAILABLE) || expire_quiescent_template(ipvs, dest) || (cdest && (dest != cdest))) { IP_VS_DBG_BUF(9, "check_template: dest not available for " @@ -2037,7 +2020,7 @@ repeat: cp = ip_vs_hn0_to_conn(hn); resched_score++; dest = cp->dest; - if (!dest || (dest->flags & IP_VS_DEST_F_AVAILABLE)) + if (!dest || (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) continue; if (atomic_read(&cp->n_control)) diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c index 0bdaeb4ed61e..eb806813292a 100644 --- a/net/netfilter/ipvs/ip_vs_core.c +++ b/net/netfilter/ipvs/ip_vs_core.c @@ -302,7 +302,7 @@ ip_vs_in_stats(struct ip_vs_conn *cp, struct sk_buff *skb) struct ip_vs_dest *dest = cp->dest; struct netns_ipvs *ipvs = cp->ipvs; - if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) { + if (dest && (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) { struct ip_vs_cpu_stats *s; struct ip_vs_service *svc; @@ -338,7 +338,7 @@ ip_vs_out_stats(struct ip_vs_conn *cp, struct sk_buff *skb) struct ip_vs_dest *dest = cp->dest; struct netns_ipvs *ipvs = cp->ipvs; - if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) { + if (dest && (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) { struct ip_vs_cpu_stats *s; struct ip_vs_service *svc; @@ -923,7 +923,7 @@ static int ip_vs_route_me_harder(struct netns_ipvs *ipvs, int af, * Packet has been made sufficiently writable in caller * - inout: 1=in->out, 0=out->in */ -void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, +bool ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, struct ip_vs_conn *cp, int inout, unsigned int toff, bool has_ports, struct ip_vs_iphdr *ciph) { @@ -931,6 +931,11 @@ void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, struct icmphdr *icmph = (struct icmphdr *)(skb->data + toff); struct iphdr *cih = (struct iphdr *)(icmph + 1); + /* Before now we may used ihl from skb frag, revalidate it after + * copying it into skb head to prevent out-of-bounds access + */ + if (cih->ihl * 4 != ciph->len - ciph->off) + return false; if (inout) { iph->saddr = cp->vaddr.ip; ip_send_check(iph); @@ -964,6 +969,7 @@ void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, else IP_VS_DBG_PKT(11, AF_INET, pp, skb, ciph->off, "Forwarding altered incoming ICMP"); + return true; } #ifdef CONFIG_IP_VS_IPV6 @@ -1055,7 +1061,8 @@ static int handle_response_icmp(int af, struct sk_buff *skb, ip_vs_nat_icmp_v6(skb, pp, cp, 1, toff, has_ports, ciph); else #endif - ip_vs_nat_icmp(skb, pp, cp, 1, toff, has_ports, ciph); + if (!ip_vs_nat_icmp(skb, pp, cp, 1, toff, has_ports, ciph)) + goto out; if (ip_vs_route_me_harder(cp->ipvs, af, skb, hooknum)) goto out; @@ -1950,6 +1957,7 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, if (pskb_pull(skb, offset2) == NULL) goto ignore_tunnel; skb_reset_network_header(skb); + memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt)); /* Ensure the IP header is present in headroom */ if (!pskb_may_pull(skb, hlen_orig)) goto ignore_tunnel; @@ -2210,7 +2218,7 @@ ip_vs_in_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state } /* Check the server status */ - if (cp && cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) { + if (cp && cp->dest && !(cp->dest->cflags & IP_VS_DEST_CF_AVAILABLE)) { /* the destination server is not available */ if (sysctl_expire_nodest_conn(ipvs)) { bool old_ct = ip_vs_conn_uses_old_conntrack(cp, skb); diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c index bcf40b8c41cf..8f9a8e491ad6 100644 --- a/net/netfilter/ipvs/ip_vs_ctl.c +++ b/net/netfilter/ipvs/ip_vs_ctl.c @@ -1304,6 +1304,40 @@ void ip_vs_stats_free(struct ip_vs_stats *stats) } } +/* Update overload flag based on number of dest conns and lower/upper + * connection thresholds: + * - conns reach u_threshold and exceed it: set the flag + * - conns go below l_threshold (or 75% of u_threshold): clear the flag + */ +static void __ip_vs_dest_update_overload(struct ip_vs_dest *dest, int mode) +{ + int conns; + u32 l, u; + + lockdep_assert_held(&dest->dst_lock); + u = READ_ONCE(dest->u_threshold); + if (!u) + goto unset; + l = READ_ONCE(dest->l_threshold_val); + conns = atomic_read(&dest->totalconns); + if (conns >= (mode > 0 ? l : u)) { + dest->flags |= IP_VS_DEST_F_OVERLOAD; + return; + } + if (conns >= (mode < 0 ? u : l)) + return; + +unset: + dest->flags &= ~IP_VS_DEST_F_OVERLOAD; +} + +void ip_vs_dest_update_overload(struct ip_vs_dest *dest, int mode) +{ + spin_lock_bh(&dest->dst_lock); + __ip_vs_dest_update_overload(dest, mode); + spin_unlock_bh(&dest->dst_lock); +} + /* * Update a destination in the given service */ @@ -1368,12 +1402,21 @@ __ip_vs_update_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, } /* set the dest status flags */ - dest->flags |= IP_VS_DEST_F_AVAILABLE; + dest->cflags |= IP_VS_DEST_CF_AVAILABLE; - if (udest->u_threshold == 0 || udest->u_threshold > dest->u_threshold) - dest->flags &= ~IP_VS_DEST_F_OVERLOAD; - dest->u_threshold = udest->u_threshold; - dest->l_threshold = udest->l_threshold; + if (READ_ONCE(dest->u_threshold) != udest->u_threshold || + READ_ONCE(dest->l_threshold) != udest->l_threshold) { + spin_lock_bh(&dest->dst_lock); + WRITE_ONCE(dest->u_threshold, udest->u_threshold); + WRITE_ONCE(dest->l_threshold, udest->l_threshold); + /* Low threshold defaults to 75% of upper threshold */ + WRITE_ONCE(dest->l_threshold_val, + udest->l_threshold ? : + (udest->u_threshold - + (udest->u_threshold >> 2))); + __ip_vs_dest_update_overload(dest, 0); + spin_unlock_bh(&dest->dst_lock); + } dest->af = udest->af; @@ -1445,7 +1488,7 @@ ip_vs_new_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) dest->port = udest->port; atomic_set(&dest->activeconns, 0); - atomic_set(&dest->inactconns, 0); + atomic_set(&dest->totalconns, 0); atomic_set(&dest->persistconns, 0); refcount_set(&dest->refcnt, 1); @@ -1486,6 +1529,9 @@ ip_vs_add_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) return -ERANGE; } + if (udest->u_threshold > INT_MAX) + return -EINVAL; + if (udest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) { if (udest->tun_port == 0) { pr_err("%s(): tunnel port is zero\n", __func__); @@ -1559,6 +1605,9 @@ ip_vs_edit_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) return -ERANGE; } + if (udest->u_threshold > INT_MAX) + return -EINVAL; + if (udest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) { if (udest->tun_port == 0) { pr_err("%s(): tunnel port is zero\n", __func__); @@ -1613,7 +1662,7 @@ static void __ip_vs_unlink_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, int svcupd) { - dest->flags &= ~IP_VS_DEST_F_AVAILABLE; + dest->cflags &= ~IP_VS_DEST_CF_AVAILABLE; spin_lock_bh(&dest->dst_lock); __ip_vs_dst_cache_reset(dest); @@ -3031,7 +3080,7 @@ static int ip_vs_info_seq_show(struct seq_file *seq, void *v) ip_vs_fwd_name(atomic_read(&dest->conn_flags)), atomic_read(&dest->weight), atomic_read(&dest->activeconns), - atomic_read(&dest->inactconns)); + ip_vs_dest_inactconns(dest)); else #endif seq_printf(seq, @@ -3042,7 +3091,7 @@ static int ip_vs_info_seq_show(struct seq_file *seq, void *v) ip_vs_fwd_name(atomic_read(&dest->conn_flags)), atomic_read(&dest->weight), atomic_read(&dest->activeconns), - atomic_read(&dest->inactconns)); + ip_vs_dest_inactconns(dest)); } } @@ -3667,10 +3716,10 @@ __ip_vs_get_dest_entries(struct netns_ipvs *ipvs, const struct ip_vs_get_dests * entry.port = dest->port; entry.conn_flags = atomic_read(&dest->conn_flags); entry.weight = atomic_read(&dest->weight); - entry.u_threshold = dest->u_threshold; - entry.l_threshold = dest->l_threshold; + entry.u_threshold = READ_ONCE(dest->u_threshold); + entry.l_threshold = READ_ONCE(dest->l_threshold); entry.activeconns = atomic_read(&dest->activeconns); - entry.inactconns = atomic_read(&dest->inactconns); + entry.inactconns = ip_vs_dest_inactconns(dest); entry.persistconns = atomic_read(&dest->persistconns); ip_vs_copy_stats(&kstats, &dest->stats); ip_vs_export_stats_user(&entry.stats, &kstats); @@ -4277,12 +4326,14 @@ static int ip_vs_genl_fill_dest(struct sk_buff *skb, struct ip_vs_dest *dest) dest->tun_port) || nla_put_u16(skb, IPVS_DEST_ATTR_TUN_FLAGS, dest->tun_flags) || - nla_put_u32(skb, IPVS_DEST_ATTR_U_THRESH, dest->u_threshold) || - nla_put_u32(skb, IPVS_DEST_ATTR_L_THRESH, dest->l_threshold) || + nla_put_u32(skb, IPVS_DEST_ATTR_U_THRESH, + READ_ONCE(dest->u_threshold)) || + nla_put_u32(skb, IPVS_DEST_ATTR_L_THRESH, + READ_ONCE(dest->l_threshold)) || nla_put_u32(skb, IPVS_DEST_ATTR_ACTIVE_CONNS, atomic_read(&dest->activeconns)) || nla_put_u32(skb, IPVS_DEST_ATTR_INACT_CONNS, - atomic_read(&dest->inactconns)) || + ip_vs_dest_inactconns(dest)) || nla_put_u32(skb, IPVS_DEST_ATTR_PERSIST_CONNS, atomic_read(&dest->persistconns)) || nla_put_u16(skb, IPVS_DEST_ATTR_ADDR_FAMILY, dest->af)) diff --git a/net/netfilter/ipvs/ip_vs_dh.c b/net/netfilter/ipvs/ip_vs_dh.c index e1f62f6b25e2..43abed7a26a6 100644 --- a/net/netfilter/ipvs/ip_vs_dh.c +++ b/net/netfilter/ipvs/ip_vs_dh.c @@ -219,8 +219,8 @@ ip_vs_dh_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, s = (struct ip_vs_dh_state *) svc->sched_data; dest = ip_vs_dh_get(svc->af, s, &iph->daddr); - if (!dest - || !(dest->flags & IP_VS_DEST_F_AVAILABLE) + if (!dest || + !(dest->cflags & IP_VS_DEST_CF_AVAILABLE) || atomic_read(&dest->weight) <= 0 || is_overloaded(dest)) { ip_vs_scheduler_err(svc, "no destination available"); diff --git a/net/netfilter/ipvs/ip_vs_lblc.c b/net/netfilter/ipvs/ip_vs_lblc.c index 15ccb2b2fa1f..693bcc82ccb7 100644 --- a/net/netfilter/ipvs/ip_vs_lblc.c +++ b/net/netfilter/ipvs/ip_vs_lblc.c @@ -502,7 +502,7 @@ ip_vs_lblc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, */ dest = en->dest; - if ((dest->flags & IP_VS_DEST_F_AVAILABLE) && + if ((dest->cflags & IP_VS_DEST_CF_AVAILABLE) && atomic_read(&dest->weight) > 0 && !is_overloaded(dest, svc)) goto out; } diff --git a/net/netfilter/ipvs/ip_vs_lblcr.c b/net/netfilter/ipvs/ip_vs_lblcr.c index c90ea897c3f7..f53f05ceea36 100644 --- a/net/netfilter/ipvs/ip_vs_lblcr.c +++ b/net/netfilter/ipvs/ip_vs_lblcr.c @@ -169,8 +169,8 @@ static inline struct ip_vs_dest *ip_vs_dest_set_min(struct ip_vs_dest_set *set) if (least->flags & IP_VS_DEST_F_OVERLOAD) continue; - if ((atomic_read(&least->weight) > 0) - && (least->flags & IP_VS_DEST_F_AVAILABLE)) { + if ((atomic_read(&least->weight) > 0) && + (least->cflags & IP_VS_DEST_CF_AVAILABLE)) { loh = ip_vs_dest_conn_overhead(least); goto nextstage; } @@ -186,8 +186,8 @@ static inline struct ip_vs_dest *ip_vs_dest_set_min(struct ip_vs_dest_set *set) doh = ip_vs_dest_conn_overhead(dest); if (((__s64)loh * atomic_read(&dest->weight) > - (__s64)doh * atomic_read(&least->weight)) - && (dest->flags & IP_VS_DEST_F_AVAILABLE)) { + (__s64)doh * atomic_read(&least->weight)) && + (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) { least = dest; loh = doh; } diff --git a/net/netfilter/ipvs/ip_vs_lc.c b/net/netfilter/ipvs/ip_vs_lc.c index 38cc38c5d8bb..18b976a7c4d2 100644 --- a/net/netfilter/ipvs/ip_vs_lc.c +++ b/net/netfilter/ipvs/ip_vs_lc.c @@ -30,7 +30,7 @@ ip_vs_lc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, /* * Simply select the server with the least number of - * (activeconns<<5) + inactconns + * (activeconns*256) + totalconns * Except whose weight is equal to zero. * If the weight is equal to zero, it means that the server is * quiesced, the existing connections to the server still get @@ -56,7 +56,7 @@ ip_vs_lc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, IP_VS_DBG_ADDR(least->af, &least->addr), ntohs(least->port), atomic_read(&least->activeconns), - atomic_read(&least->inactconns)); + ip_vs_dest_inactconns(least)); return least; } diff --git a/net/netfilter/ipvs/ip_vs_proto_sctp.c b/net/netfilter/ipvs/ip_vs_proto_sctp.c index c80567c73469..fb8af6b15a39 100644 --- a/net/netfilter/ipvs/ip_vs_proto_sctp.c +++ b/net/netfilter/ipvs/ip_vs_proto_sctp.c @@ -446,12 +446,10 @@ set_sctp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp, if (!(cp->flags & IP_VS_CONN_F_INACTIVE) && (next_state != IP_VS_SCTP_S_ESTABLISHED)) { atomic_dec(&dest->activeconns); - atomic_inc(&dest->inactconns); cp->flags |= IP_VS_CONN_F_INACTIVE; } else if ((cp->flags & IP_VS_CONN_F_INACTIVE) && (next_state == IP_VS_SCTP_S_ESTABLISHED)) { atomic_inc(&dest->activeconns); - atomic_dec(&dest->inactconns); cp->flags &= ~IP_VS_CONN_F_INACTIVE; } } diff --git a/net/netfilter/ipvs/ip_vs_proto_tcp.c b/net/netfilter/ipvs/ip_vs_proto_tcp.c index 99a286fdc90c..fec0e8b47b71 100644 --- a/net/netfilter/ipvs/ip_vs_proto_tcp.c +++ b/net/netfilter/ipvs/ip_vs_proto_tcp.c @@ -526,12 +526,10 @@ set_tcp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp, if (!(cp->flags & IP_VS_CONN_F_INACTIVE) && !tcp_state_active(new_state)) { atomic_dec(&dest->activeconns); - atomic_inc(&dest->inactconns); cp->flags |= IP_VS_CONN_F_INACTIVE; } else if ((cp->flags & IP_VS_CONN_F_INACTIVE) && tcp_state_active(new_state)) { atomic_inc(&dest->activeconns); - atomic_dec(&dest->inactconns); cp->flags &= ~IP_VS_CONN_F_INACTIVE; } } diff --git a/net/netfilter/ipvs/ip_vs_sync.c b/net/netfilter/ipvs/ip_vs_sync.c index 93038abbf5e0..ea5fdd4f4ce7 100644 --- a/net/netfilter/ipvs/ip_vs_sync.c +++ b/net/netfilter/ipvs/ip_vs_sync.c @@ -878,13 +878,10 @@ static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *pa spin_lock_bh(&cp->lock); if ((cp->flags ^ flags) & IP_VS_CONN_F_INACTIVE && !(flags & IP_VS_CONN_F_TEMPLATE) && dest) { - if (flags & IP_VS_CONN_F_INACTIVE) { + if (flags & IP_VS_CONN_F_INACTIVE) atomic_dec(&dest->activeconns); - atomic_inc(&dest->inactconns); - } else { + else atomic_inc(&dest->activeconns); - atomic_dec(&dest->inactconns); - } } flags &= IP_VS_CONN_F_BACKUP_UPD_MASK; flags |= cp->flags & ~IP_VS_CONN_F_BACKUP_UPD_MASK; diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c index c4508f3f43dd..04450a48f01a 100644 --- a/net/netfilter/ipvs/ip_vs_xmit.c +++ b/net/netfilter/ipvs/ip_vs_xmit.c @@ -351,7 +351,7 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, * stored in dest_trash. */ if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) && - dest->flags & IP_VS_DEST_F_AVAILABLE) + dest->cflags & IP_VS_DEST_CF_AVAILABLE) __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0); else noref = 0; @@ -530,7 +530,7 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, * stored in dest_trash. */ if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) && - dest->flags & IP_VS_DEST_F_AVAILABLE) + dest->cflags & IP_VS_DEST_CF_AVAILABLE) __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie); else noref = 0; @@ -1580,7 +1580,8 @@ ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, if (skb_cow(skb, rt->dst.dev->hard_header_len)) goto tx_error; - ip_vs_nat_icmp(skb, pp, cp, 0, toff, has_ports, ciph); + if (!ip_vs_nat_icmp(skb, pp, cp, 0, toff, has_ports, ciph)) + goto tx_error; /* Another hack: avoid icmp_send in ip_fragment */ skb->ignore_df = 1; diff --git a/net/netfilter/nf_conntrack_proto.c b/net/netfilter/nf_conntrack_proto.c index ad96896516b6..7a40e4e0e33e 100644 --- a/net/netfilter/nf_conntrack_proto.c +++ b/net/netfilter/nf_conntrack_proto.c @@ -79,6 +79,12 @@ void nf_ct_l4proto_log_invalid(const struct sk_buff *skb, struct net *net; va_list args; + /* nfnetlink_log may re-enter conntrack attribute dumping and try to + * take ct->lock again via helpers such as tcp_to_nlattr(), so invalid + * conntrack logs must only be emitted after dropping ct->lock. + */ + lockdep_assert_not_held(&ct->lock); + net = nf_ct_net(ct); if (likely(net->ct.sysctl_log_invalid == 0)) return; diff --git a/net/netfilter/nf_conntrack_proto_sctp.c b/net/netfilter/nf_conntrack_proto_sctp.c index 7e10fa65cbdd..71cc920f6856 100644 --- a/net/netfilter/nf_conntrack_proto_sctp.c +++ b/net/netfilter/nf_conntrack_proto_sctp.c @@ -336,10 +336,12 @@ int nf_conntrack_sctp_packet(struct nf_conn *ct, struct sctphdr _sctph; const struct sctp_chunkhdr *sch; struct sctp_chunkhdr _sch; + bool log_invalid = false; u_int32_t offset, count; unsigned int *timeouts; unsigned long map[256 / sizeof(unsigned long)] = { 0 }; bool ignore = false; + u8 invalid_type = 0; if (sctp_error(skb, dataoff, state)) return -NF_ACCEPT; @@ -451,10 +453,8 @@ int nf_conntrack_sctp_packet(struct nf_conn *ct, /* Invalid */ if (new_state == SCTP_CONNTRACK_MAX) { - nf_ct_l4proto_log_invalid(skb, ct, state, - "Invalid, old_state %d, dir %d, type %d", - old_state, dir, sch->type); - + log_invalid = true; + invalid_type = sch->type; goto out_unlock; } @@ -529,6 +529,10 @@ int nf_conntrack_sctp_packet(struct nf_conn *ct, out_unlock: spin_unlock_bh(&ct->lock); + if (log_invalid) + nf_ct_l4proto_log_invalid(skb, ct, state, + "Invalid, old_state %d, dir %d, type %d", + old_state, dir, invalid_type); out: return -NF_ACCEPT; } diff --git a/net/netfilter/nf_conntrack_proto_tcp.c b/net/netfilter/nf_conntrack_proto_tcp.c index ceeed3d7fe52..30b970e2ade5 100644 --- a/net/netfilter/nf_conntrack_proto_tcp.c +++ b/net/netfilter/nf_conntrack_proto_tcp.c @@ -480,37 +480,81 @@ static void tcp_init_sender(struct ip_ct_tcp_state *sender, } } -__printf(6, 7) -static enum nf_ct_tcp_action nf_tcp_log_invalid(const struct sk_buff *skb, - const struct nf_conn *ct, - const struct nf_hook_state *state, - const struct ip_ct_tcp_state *sender, - enum nf_ct_tcp_action ret, - const char *fmt, ...) +enum nf_tcp_invalid_log_type { + NF_TCP_LOG_NONE, + NF_TCP_LOG_OVERSHOT, + NF_TCP_LOG_SEQ_OVER, + NF_TCP_LOG_ACK_OVER, + NF_TCP_LOG_SEQ_UNDER, + NF_TCP_LOG_ACK_UNDER, +}; + +struct nf_tcp_invalid_log { + enum nf_tcp_invalid_log_type type; + u32 value; +}; + +static enum nf_ct_tcp_action +nf_tcp_store_invalid(const struct nf_conn *ct, + const struct ip_ct_tcp_state *sender, + struct nf_tcp_invalid_log *log, + enum nf_ct_tcp_action ret, + enum nf_tcp_invalid_log_type type, + u32 value) { const struct nf_tcp_net *tn = nf_tcp_pernet(nf_ct_net(ct)); - struct va_format vaf; - va_list args; bool be_liberal; be_liberal = sender->flags & IP_CT_TCP_FLAG_BE_LIBERAL || tn->tcp_be_liberal; if (be_liberal) return NFCT_TCP_ACCEPT; - va_start(args, fmt); - vaf.fmt = fmt; - vaf.va = &args; - nf_ct_l4proto_log_invalid(skb, ct, state, "%pV", &vaf); - va_end(args); - + log->type = type; + log->value = value; return ret; } +static void nf_tcp_log_invalid(const struct sk_buff *skb, + const struct nf_conn *ct, + const struct nf_hook_state *state, + const struct nf_tcp_invalid_log *log) +{ + switch (log->type) { + case NF_TCP_LOG_OVERSHOT: + nf_ct_l4proto_log_invalid(skb, ct, state, + "%u bytes more than expected", + log->value); + break; + case NF_TCP_LOG_SEQ_OVER: + nf_ct_l4proto_log_invalid(skb, ct, state, + "SEQ is over upper bound %u (over the window of the receiver)", + log->value); + break; + case NF_TCP_LOG_ACK_OVER: + nf_ct_l4proto_log_invalid(skb, ct, state, + "ACK is over upper bound %u (ACKed data not seen yet)", + log->value); + break; + case NF_TCP_LOG_SEQ_UNDER: + nf_ct_l4proto_log_invalid(skb, ct, state, + "SEQ is under lower bound %u (already ACKed data retransmitted)", + log->value); + break; + case NF_TCP_LOG_ACK_UNDER: + nf_ct_l4proto_log_invalid(skb, ct, state, + "ignored ACK under lower bound %u (possible overly delayed)", + log->value); + break; + case NF_TCP_LOG_NONE: + break; + } +} + static enum nf_ct_tcp_action tcp_in_window(struct nf_conn *ct, enum ip_conntrack_dir dir, unsigned int index, const struct sk_buff *skb, unsigned int dataoff, const struct tcphdr *tcph, - const struct nf_hook_state *hook_state) + struct nf_tcp_invalid_log *log) { struct ip_ct_tcp *state = &ct->proto.tcp; struct ip_ct_tcp_state *sender = &state->seen[dir]; @@ -640,31 +684,29 @@ tcp_in_window(struct nf_conn *ct, enum ip_conntrack_dir dir, sender->td_end = end; sender->flags |= IP_CT_TCP_FLAG_DATA_UNACKNOWLEDGED; - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE, - "%u bytes more than expected", overshot); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_IGNORE, + NF_TCP_LOG_OVERSHOT, overshot); } - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_INVALID, - "SEQ is over upper bound %u (over the window of the receiver)", - sender->td_maxend + 1); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_INVALID, + NF_TCP_LOG_SEQ_OVER, sender->td_maxend + 1); } if (!before(sack, receiver->td_end + 1)) - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_INVALID, - "ACK is over upper bound %u (ACKed data not seen yet)", - receiver->td_end + 1); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_INVALID, + NF_TCP_LOG_ACK_OVER, receiver->td_end + 1); /* Is the ending sequence in the receive window (if available)? */ in_recv_win = !receiver->td_maxwin || after(end, sender->td_end - receiver->td_maxwin - 1); if (!in_recv_win) - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE, - "SEQ is under lower bound %u (already ACKed data retransmitted)", - sender->td_end - receiver->td_maxwin - 1); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_IGNORE, + NF_TCP_LOG_SEQ_UNDER, + sender->td_end - receiver->td_maxwin - 1); if (!after(sack, receiver->td_end - MAXACKWINDOW(sender) - 1)) - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE, - "ignored ACK under lower bound %u (possible overly delayed)", - receiver->td_end - MAXACKWINDOW(sender) - 1); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_IGNORE, + NF_TCP_LOG_ACK_UNDER, + receiver->td_end - MAXACKWINDOW(sender) - 1); /* Take into account window scaling (RFC 1323). */ if (!tcph->syn) @@ -719,11 +761,8 @@ tcp_in_window(struct nf_conn *ct, enum ip_conntrack_dir dir, return NFCT_TCP_ACCEPT; } -static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, - enum ip_conntrack_dir dir, - int index, - const struct sk_buff *skb, - const struct nf_hook_state *hook_state) +static bool __cold +nf_tcp_handle_invalid(struct nf_conn *ct, enum ip_conntrack_dir dir, int index) { const unsigned int *timeouts; const struct nf_tcp_net *tn; @@ -732,7 +771,7 @@ static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, if (!test_bit(IPS_ASSURED_BIT, &ct->status) || test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status)) - return; + return false; /* We don't want to have connections hanging around in ESTABLISHED * state for long time 'just because' conntrack deemed a FIN/RST @@ -747,7 +786,7 @@ static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, case TCP_FIN_SET: break; default: - return; + return false; } if (ct->proto.tcp.last_dir != dir && @@ -755,7 +794,7 @@ static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, ct->proto.tcp.last_index == TCP_RST_SET)) { expires = nf_ct_expires(ct); if (expires < 120 * HZ) - return; + return false; tn = nf_tcp_pernet(nf_ct_net(ct)); timeouts = nf_ct_timeout_lookup(ct); @@ -764,16 +803,15 @@ static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, timeout = READ_ONCE(timeouts[TCP_CONNTRACK_UNACK]); if (expires > timeout) { - nf_ct_l4proto_log_invalid(skb, ct, hook_state, - "packet (index %d, dir %d) response for index %d lower timeout to %u", - index, dir, ct->proto.tcp.last_index, timeout); - WRITE_ONCE(ct->timeout, timeout + nfct_time_stamp); + return true; } } else { ct->proto.tcp.last_index = index; ct->proto.tcp.last_dir = dir; } + + return false; } /* table of valid flag combinations - PUSH, ECE and CWR are always valid */ @@ -969,7 +1007,9 @@ int nf_conntrack_tcp_packet(struct nf_conn *ct, struct net *net = nf_ct_net(ct); struct nf_tcp_net *tn = nf_tcp_pernet(net); enum tcp_conntrack new_state, old_state; + struct nf_tcp_invalid_log log = {}; unsigned int index, *timeouts; + bool lowered_timeout = false; enum nf_ct_tcp_action res; enum ip_conntrack_dir dir; const struct tcphdr *th; @@ -1252,14 +1292,18 @@ int nf_conntrack_tcp_packet(struct nf_conn *ct, } res = tcp_in_window(ct, dir, index, - skb, dataoff, th, state); + skb, dataoff, th, &log); switch (res) { case NFCT_TCP_IGNORE: spin_unlock_bh(&ct->lock); + nf_tcp_log_invalid(skb, ct, state, &log); return NF_ACCEPT; case NFCT_TCP_INVALID: - nf_tcp_handle_invalid(ct, dir, index, skb, state); + lowered_timeout = nf_tcp_handle_invalid(ct, dir, index); spin_unlock_bh(&ct->lock); + nf_tcp_log_invalid(skb, ct, state, &log); + if (lowered_timeout) + nf_ct_l4proto_log_invalid(skb, ct, state, "lowered timeout to UNACK"); return -NF_ACCEPT; case NFCT_TCP_ACCEPT: break; diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c index b66e65439341..8b1165f2b5a4 100644 --- a/net/netfilter/nf_flow_table_core.c +++ b/net/netfilter/nf_flow_table_core.c @@ -332,17 +332,18 @@ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow) flow->timeout = nf_flowtable_time_stamp + flow_offload_get_timeout(flow); err = rhashtable_insert_fast(&flow_table->rhashtable, - &flow->tuplehash[0].node, + &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node, nf_flow_offload_rhash_params); if (err < 0) return err; + /* GC only iterates original-direction entries; publish original last. */ err = rhashtable_insert_fast(&flow_table->rhashtable, - &flow->tuplehash[1].node, + &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node, nf_flow_offload_rhash_params); if (err < 0) { rhashtable_remove_fast(&flow_table->rhashtable, - &flow->tuplehash[0].node, + &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node, nf_flow_offload_rhash_params); return err; } diff --git a/net/netfilter/nf_log_syslog.c b/net/netfilter/nf_log_syslog.c index e37b09b3203b..5ffde27d450a 100644 --- a/net/netfilter/nf_log_syslog.c +++ b/net/netfilter/nf_log_syslog.c @@ -1014,7 +1014,7 @@ err1: return ret; } -static void __net_exit nf_log_syslog_net_exit(struct net *net) +static void __net_exit nf_log_syslog_net_pre_exit(struct net *net) { nf_log_unset(net, &nf_ip_logger); nf_log_unset(net, &nf_arp_logger); @@ -1025,7 +1025,7 @@ static void __net_exit nf_log_syslog_net_exit(struct net *net) static struct pernet_operations nf_log_syslog_net_ops = { .init = nf_log_syslog_net_init, - .exit = nf_log_syslog_net_exit, + .pre_exit = nf_log_syslog_net_pre_exit, }; static int __init nf_log_syslog_init(void) diff --git a/net/netfilter/nf_tables_offload.c b/net/netfilter/nf_tables_offload.c index 8998a24651ff..0ac3c26dfb3d 100644 --- a/net/netfilter/nf_tables_offload.c +++ b/net/netfilter/nf_tables_offload.c @@ -558,7 +558,7 @@ static void nft_flow_rule_offload_abort(struct net *net, break; } - if (WARN_ON_ONCE(err)) + if (WARN_ON_ONCE(err && err != -ENOMEM)) break; } } diff --git a/net/netfilter/nfnetlink_log.c b/net/netfilter/nfnetlink_log.c index 5fee61b3813c..6c7fa2ed34f5 100644 --- a/net/netfilter/nfnetlink_log.c +++ b/net/netfilter/nfnetlink_log.c @@ -1170,21 +1170,26 @@ static int __net_init nfnl_log_net_init(struct net *net) return 0; } +static void __net_exit nfnl_log_net_pre_exit(struct net *net) +{ +#ifdef CONFIG_PROC_FS + remove_proc_entry("nfnetlink_log", net->nf.proc_netfilter); +#endif + nf_log_unset(net, &nfulnl_logger); +} + static void __net_exit nfnl_log_net_exit(struct net *net) { struct nfnl_log_net *log = nfnl_log_pernet(net); unsigned int i; -#ifdef CONFIG_PROC_FS - remove_proc_entry("nfnetlink_log", net->nf.proc_netfilter); -#endif - nf_log_unset(net, &nfulnl_logger); for (i = 0; i < INSTANCE_BUCKETS; i++) WARN_ON_ONCE(!hlist_empty(&log->instance_table[i])); } static struct pernet_operations nfnl_log_net_ops = { .init = nfnl_log_net_init, + .pre_exit = nfnl_log_net_pre_exit, .exit = nfnl_log_net_exit, .id = &nfnl_log_net_id, .size = sizeof(struct nfnl_log_net),