diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h index b99805ee2fd1..d1f0a427e9c8 100644 --- a/include/net/ip6_tunnel.h +++ b/include/net/ip6_tunnel.h @@ -45,6 +45,7 @@ struct __ip6_tnl_parm { /* IPv6 tunnel */ struct ip6_tnl { struct ip6_tnl __rcu *next; /* next tunnel in list */ + struct hlist_node hash_node; struct net_device *dev; /* virtual device associated with tunnel */ netdevice_tracker dev_tracker; struct net *net; /* netns for packet i/o */ diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 8ebda0b6a78b..c851af22b9fe 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -64,7 +64,8 @@ MODULE_PARM_DESC(log_ecn_error, "Log packets received with corrupted ECN"); static unsigned int ip6gre_net_id __read_mostly; struct ip6gre_net { - struct ip6_tnl __rcu *tunnels[4][IP6_GRE_HASH_SIZE]; + struct hlist_head tunnels[4][IP6_GRE_HASH_SIZE]; + struct mutex tunnels_lock; struct ip6_tnl __rcu *collect_md_tun; struct ip6_tnl __rcu *collect_md_tun_erspan; @@ -79,6 +80,8 @@ static void ip6gre_tunnel_setup(struct net_device *dev); static void ip6gre_tunnel_link(struct ip6gre_net *ign, struct ip6_tnl *t); static void ip6gre_tnl_link_config(struct ip6_tnl *t, int set_mtu); static void ip6erspan_tnl_link_config(struct ip6_tnl *t, int set_mtu); +static void __ip6gre_dellink(struct net *net, struct net_device *dev, + struct list_head *head); /* Tunnel hash table */ @@ -141,20 +144,24 @@ static struct ip6_tnl *ip6gre_tunnel_lookup(struct net_device *dev, const struct in6_addr *remote, const struct in6_addr *local, __be32 key, __be16 gre_proto) { - struct net *net = dev_net(dev); - int link = dev->ifindex; - unsigned int h0 = HASH_ADDR(remote); - unsigned int h1 = HASH_KEY(key); - struct ip6_tnl *t, *cand = NULL; - struct ip6gre_net *ign = net_generic(net, ip6gre_net_id); int dev_type = (gre_proto == htons(ETH_P_TEB) || gre_proto == htons(ETH_P_ERSPAN) || gre_proto == htons(ETH_P_ERSPAN2)) ? ARPHRD_ETHER : ARPHRD_IP6GRE; + unsigned int h0 = HASH_ADDR(remote); + unsigned int h1 = HASH_KEY(key); + struct ip6_tnl *t, *cand = NULL; + struct net *net = dev_net(dev); struct net_device *ndev; + struct hlist_head *head; + int link = dev->ifindex; + struct ip6gre_net *ign; int cand_score = 4; - for_each_ip_tunnel_rcu(t, ign->tunnels_r_l[h0 ^ h1]) { + ign = net_generic(net, ip6gre_net_id); + + head = &ign->tunnels_r_l[h0 ^ h1]; + hlist_for_each_entry_rcu(t, head, hash_node) { if (!ipv6_addr_equal(local, &t->parms.laddr) || !ipv6_addr_equal(remote, &t->parms.raddr) || key != t->parms.i_key || @@ -165,7 +172,8 @@ static struct ip6_tnl *ip6gre_tunnel_lookup(struct net_device *dev, return cand; } - for_each_ip_tunnel_rcu(t, ign->tunnels_r[h0 ^ h1]) { + head = &ign->tunnels_r[h0 ^ h1]; + hlist_for_each_entry_rcu(t, head, hash_node) { if (!ipv6_addr_equal(remote, &t->parms.raddr) || key != t->parms.i_key || !(t->dev->flags & IFF_UP)) @@ -175,7 +183,8 @@ static struct ip6_tnl *ip6gre_tunnel_lookup(struct net_device *dev, return cand; } - for_each_ip_tunnel_rcu(t, ign->tunnels_l[h1]) { + head = &ign->tunnels_l[h1]; + hlist_for_each_entry_rcu(t, head, hash_node) { if ((!ipv6_addr_equal(local, &t->parms.laddr) && (!ipv6_addr_equal(local, &t->parms.raddr) || !ipv6_addr_is_multicast(local))) || @@ -187,7 +196,8 @@ static struct ip6_tnl *ip6gre_tunnel_lookup(struct net_device *dev, return cand; } - for_each_ip_tunnel_rcu(t, ign->tunnels_wc[h1]) { + head = &ign->tunnels_wc[h1]; + hlist_for_each_entry_rcu(t, head, hash_node) { if (t->parms.i_key != key || !(t->dev->flags & IFF_UP)) continue; @@ -215,8 +225,8 @@ static struct ip6_tnl *ip6gre_tunnel_lookup(struct net_device *dev, return NULL; } -static struct ip6_tnl __rcu **__ip6gre_bucket(struct ip6gre_net *ign, - const struct __ip6_tnl_parm *p) +static struct hlist_head *__ip6gre_bucket(struct ip6gre_net *ign, + const struct __ip6_tnl_parm *p) { const struct in6_addr *remote = &p->raddr; const struct in6_addr *local = &p->laddr; @@ -258,56 +268,51 @@ static void ip6erspan_tunnel_unlink_md(struct ip6gre_net *ign, rcu_assign_pointer(ign->collect_md_tun_erspan, NULL); } -static inline struct ip6_tnl __rcu **ip6gre_bucket(struct ip6gre_net *ign, - const struct ip6_tnl *t) +static inline struct hlist_head *ip6gre_bucket(struct ip6gre_net *ign, + const struct ip6_tnl *t) { return __ip6gre_bucket(ign, &t->parms); } static void ip6gre_tunnel_link(struct ip6gre_net *ign, struct ip6_tnl *t) { - struct ip6_tnl __rcu **tp = ip6gre_bucket(ign, t); + struct hlist_head *head = ip6gre_bucket(ign, t); - rcu_assign_pointer(t->next, rtnl_dereference(*tp)); - rcu_assign_pointer(*tp, t); + hlist_add_head_rcu(&t->hash_node, head); } static void ip6gre_tunnel_unlink(struct ip6gre_net *ign, struct ip6_tnl *t) { - struct ip6_tnl __rcu **tp; - struct ip6_tnl *iter; + hlist_del_init_rcu(&t->hash_node); +} - for (tp = ip6gre_bucket(ign, t); - (iter = rtnl_dereference(*tp)) != NULL; - tp = &iter->next) { - if (t == iter) { - rcu_assign_pointer(*tp, t->next); - break; - } - } +static bool ip6gre_tunnel_unregistering(struct ip6_tnl *t) +{ + return hlist_unhashed(&t->hash_node); } static struct ip6_tnl *ip6gre_tunnel_find(struct net *net, const struct __ip6_tnl_parm *parms, int type) { + struct ip6gre_net *ign = net_generic(net, ip6gre_net_id); const struct in6_addr *remote = &parms->raddr; const struct in6_addr *local = &parms->laddr; __be32 key = parms->i_key; + struct hlist_head *head; int link = parms->link; struct ip6_tnl *t; - struct ip6_tnl __rcu **tp; - struct ip6gre_net *ign = net_generic(net, ip6gre_net_id); - for (tp = __ip6gre_bucket(ign, parms); - (t = rtnl_dereference(*tp)) != NULL; - tp = &t->next) + head = __ip6gre_bucket(ign, parms); + + hlist_for_each_entry(t, head, hash_node) { if (ipv6_addr_equal(local, &t->parms.laddr) && ipv6_addr_equal(remote, &t->parms.raddr) && key == t->parms.i_key && link == t->parms.link && type == t->dev->type) break; + } return t; } @@ -362,10 +367,7 @@ failed_free: static void ip6erspan_tunnel_uninit(struct net_device *dev) { struct ip6_tnl *t = netdev_priv(dev); - struct ip6gre_net *ign = net_generic(t->net, ip6gre_net_id); - ip6erspan_tunnel_unlink_md(ign, t); - ip6gre_tunnel_unlink(ign, t); dst_cache_reset(&t->dst_cache); netdev_put(dev, &t->dev_tracker); } @@ -373,12 +375,7 @@ static void ip6erspan_tunnel_uninit(struct net_device *dev) static void ip6gre_tunnel_uninit(struct net_device *dev) { struct ip6_tnl *t = netdev_priv(dev); - struct ip6gre_net *ign = net_generic(t->net, ip6gre_net_id); - ip6gre_tunnel_unlink_md(ign, t); - ip6gre_tunnel_unlink(ign, t); - if (ign->fb_tunnel_dev == dev) - WRITE_ONCE(ign->fb_tunnel_dev, NULL); dst_cache_reset(&t->dst_cache); netdev_put(dev, &t->dev_tracker); } @@ -1256,14 +1253,25 @@ static int ip6gre_tunnel_siocdevprivate(struct net_device *dev, struct ifreq *ifr, void __user *data, int cmd) { - int err = 0; - struct ip6_tnl_parm2 p; - struct __ip6_tnl_parm p1; struct ip6_tnl *t = netdev_priv(dev); + struct net *orig_net = dev_net(dev); + struct __ip6_tnl_parm p1 = {}; + LIST_HEAD(dev_kill_list); struct net *net = t->net; - struct ip6gre_net *ign = net_generic(net, ip6gre_net_id); + struct ip6_tnl_parm2 p; + struct ip6gre_net *ign; + int err = 0; - memset(&p1, 0, sizeof(p1)); + DEBUG_NET_WARN_ON_ONCE(netdev_need_ops_lock(dev)); + + ign = net_generic(net, ip6gre_net_id); + + mutex_lock(&ign->tunnels_lock); + + if (!check_net(net)) { + err = -EBUSY; + goto done; + } switch (cmd) { case SIOCGETTUNNEL: @@ -1306,7 +1314,7 @@ static int ip6gre_tunnel_siocdevprivate(struct net_device *dev, t = ip6gre_tunnel_locate(net, &p1, cmd == SIOCADDTUNNEL); if (dev != ign->fb_tunnel_dev && cmd == SIOCCHGTUNNEL) { - if (t) { + if (t && !ip6gre_tunnel_unregistering(t)) { if (t->dev != dev) { err = -EEXIST; break; @@ -1314,23 +1322,26 @@ static int ip6gre_tunnel_siocdevprivate(struct net_device *dev, } else { t = netdev_priv(dev); - ip6gre_tunnel_unlink(ign, t); - synchronize_net(); - ip6gre_tnl_change(t, &p1, 1); - ip6gre_tunnel_link(ign, t); - netdev_state_change(dev); + if (!ip6gre_tunnel_unregistering(t)) { + ip6gre_tunnel_unlink(ign, t); + synchronize_net(); + ip6gre_tnl_change(t, &p1, 1); + ip6gre_tunnel_link(ign, t); + netdev_state_change(dev); + } } } - if (t) { + if (t && !ip6gre_tunnel_unregistering(t)) { err = 0; memset(&p, 0, sizeof(p)); ip6gre_tnl_parm_to_user(&p, &t->parms); if (copy_to_user(data, &p, sizeof(p))) err = -EFAULT; - } else + } else { err = (cmd == SIOCADDTUNNEL ? -ENOBUFS : -ENOENT); + } break; case SIOCDELTUNNEL: @@ -1352,7 +1363,9 @@ static int ip6gre_tunnel_siocdevprivate(struct net_device *dev, goto done; dev = t->dev; } - unregister_netdevice(dev); + + if (!ip6gre_tunnel_unregistering(t)) + __ip6gre_dellink(orig_net, dev, &dev_kill_list); err = 0; break; @@ -1361,6 +1374,9 @@ static int ip6gre_tunnel_siocdevprivate(struct net_device *dev, } done: + mutex_unlock(&ign->tunnels_lock); + + unregister_netdevice_many(&dev_kill_list); return err; } @@ -1551,44 +1567,47 @@ static struct inet6_protocol ip6gre_protocol __read_mostly = { .flags = INET6_PROTO_FINAL, }; -static void __net_exit ip6gre_exit_rtnl_net(struct net *net, struct list_head *head) +static void __net_exit ip6gre_exit_rtnl_net(struct net *net, + struct list_head *dev_kill_list) { struct ip6gre_net *ign = net_generic(net, ip6gre_net_id); - struct net_device *dev, *aux; int prio; - for_each_netdev_safe(net, dev, aux) - if (dev->rtnl_link_ops == &ip6gre_link_ops || - dev->rtnl_link_ops == &ip6gre_tap_ops || - dev->rtnl_link_ops == &ip6erspan_tap_ops) - unregister_netdevice_queue(dev, head); + WRITE_ONCE(ign->fb_tunnel_dev, NULL); + + mutex_lock(&ign->tunnels_lock); for (prio = 0; prio < 4; prio++) { int h; + for (h = 0; h < IP6_GRE_HASH_SIZE; h++) { + struct hlist_head *head = &ign->tunnels[prio][h]; + struct hlist_node *tmp; struct ip6_tnl *t; - t = rtnl_net_dereference(net, ign->tunnels[prio][h]); - - while (t) { - /* If dev is in the same netns, it has already - * been added to the list by the previous loop. - */ - if (!net_eq(dev_net(t->dev), net)) - unregister_netdevice_queue(t->dev, head); - - t = rtnl_net_dereference(net, t->next); - } + hlist_for_each_entry_safe(t, tmp, head, hash_node) + __ip6gre_dellink(net, t->dev, dev_kill_list); } } + + mutex_unlock(&ign->tunnels_lock); } static int __net_init ip6gre_init_net(struct net *net) { struct ip6gre_net *ign = net_generic(net, ip6gre_net_id); struct net_device *ndev; + struct ip6_tnl *t; + int prio, h; int err; + for (prio = 0; prio < 4; prio++) { + for (h = 0; h < IP6_GRE_HASH_SIZE; h++) + INIT_HLIST_HEAD(&ign->tunnels[prio][h]); + } + + mutex_init(&ign->tunnels_lock); + if (!net_has_fallback_tunnels(net)) return 0; ndev = alloc_netdev(sizeof(struct ip6_tnl), "ip6gre0", @@ -1607,12 +1626,13 @@ static int __net_init ip6gre_init_net(struct net *net) ip6gre_fb_tunnel_init(ign->fb_tunnel_dev); ign->fb_tunnel_dev->rtnl_link_ops = &ip6gre_link_ops; + t = netdev_priv(ign->fb_tunnel_dev); + hlist_add_head_rcu(&t->hash_node, &ign->tunnels_wc[0]); + err = register_netdev(ign->fb_tunnel_dev); if (err) goto err_reg_dev; - rcu_assign_pointer(ign->tunnels_wc[0], - netdev_priv(ign->fb_tunnel_dev)); return 0; err_reg_dev: @@ -1984,18 +2004,22 @@ static int ip6gre_newlink(struct net_device *dev, struct nlattr **data = params->data; struct nlattr **tb = params->tb; struct ip6gre_net *ign; - int err; + int err = 0; ip6gre_netlink_parms(data, &nt->parms); ign = net_generic(net, ip6gre_net_id); + mutex_lock(&ign->tunnels_lock); + if (nt->parms.collect_md) { if (rtnl_dereference(ign->collect_md_tun)) - return -EEXIST; + err = -EEXIST; } else { if (ip6gre_tunnel_find(net, &nt->parms, dev->type)) - return -EEXIST; + err = -EEXIST; } + if (err) + goto unlock; err = ip6gre_newlink_common(net, dev, tb, data, extack); if (!err) { @@ -2003,6 +2027,10 @@ static int ip6gre_newlink(struct net_device *dev, ip6gre_tunnel_link_md(ign, nt); ip6gre_tunnel_link(net_generic(net, ip6gre_net_id), nt); } + +unlock: + mutex_unlock(&ign->tunnels_lock); + return err; } @@ -2037,6 +2065,9 @@ ip6gre_changelink_common(struct net_device *dev, struct nlattr *tb[], t = nt; } + if (ip6gre_tunnel_unregistering(t)) + return ERR_PTR(-ENODEV); + return t; } @@ -2045,31 +2076,65 @@ static int ip6gre_changelink(struct net_device *dev, struct nlattr *tb[], struct netlink_ext_ack *extack) { struct ip6_tnl *t = netdev_priv(dev); - struct ip6gre_net *ign = net_generic(t->net, ip6gre_net_id); struct __ip6_tnl_parm p; + struct ip6gre_net *ign; + int err = 0; + + ign = net_generic(t->net, ip6gre_net_id); if (!rtnl_dev_link_net_capable(dev, t->net)) return -EPERM; + mutex_lock(&ign->tunnels_lock); + t = ip6gre_changelink_common(dev, tb, data, &p, extack); - if (IS_ERR(t)) - return PTR_ERR(t); + if (IS_ERR(t)) { + err = PTR_ERR(t); + goto unlock; + } ip6gre_tunnel_unlink_md(ign, t); ip6gre_tunnel_unlink(ign, t); ip6gre_tnl_change(t, &p, !tb[IFLA_MTU]); ip6gre_tunnel_link_md(ign, t); ip6gre_tunnel_link(ign, t); - return 0; +unlock: + mutex_unlock(&ign->tunnels_lock); + + return err; +} + +static void __ip6gre_dellink(struct net *net, struct net_device *dev, + struct list_head *head) +{ + struct ip6_tnl *t = netdev_priv(dev); + struct ip6gre_net *ign; + + ign = net_generic(t->net, ip6gre_net_id); + + if (dev->rtnl_link_ops == &ip6erspan_tap_ops) + ip6erspan_tunnel_unlink_md(ign, t); + else + ip6gre_tunnel_unlink_md(ign, t); + + ip6gre_tunnel_unlink(ign, t); + unregister_netdevice_queue_net(net, dev, head); } static void ip6gre_dellink(struct net_device *dev, struct list_head *head) { - struct net *net = dev_net(dev); - struct ip6gre_net *ign = net_generic(net, ip6gre_net_id); + struct ip6_tnl *t = netdev_priv(dev); + struct ip6gre_net *ign; - if (dev != ign->fb_tunnel_dev) - unregister_netdevice_queue(dev, head); + ign = net_generic(t->net, ip6gre_net_id); + + mutex_lock(&ign->tunnels_lock); + + if (dev != ign->fb_tunnel_dev && + !ip6gre_tunnel_unregistering(t)) + __ip6gre_dellink(dev_net(dev), dev, head); + + mutex_unlock(&ign->tunnels_lock); } static size_t ip6gre_get_size(const struct net_device *dev) @@ -2225,19 +2290,23 @@ static int ip6erspan_newlink(struct net_device *dev, struct nlattr **data = params->data; struct nlattr **tb = params->tb; struct ip6gre_net *ign; - int err; + int err = 0; ip6gre_netlink_parms(data, &nt->parms); ip6erspan_set_version(data, &nt->parms); ign = net_generic(net, ip6gre_net_id); + mutex_lock(&ign->tunnels_lock); + if (nt->parms.collect_md) { if (rtnl_dereference(ign->collect_md_tun_erspan)) - return -EEXIST; + err = -EEXIST; } else { if (ip6gre_tunnel_find(net, &nt->parms, dev->type)) - return -EEXIST; + err = -EEXIST; } + if (err) + goto unlock; err = ip6gre_newlink_common(net, dev, tb, data, extack); if (!err) { @@ -2245,6 +2314,10 @@ static int ip6erspan_newlink(struct net_device *dev, ip6erspan_tunnel_link_md(ign, nt); ip6gre_tunnel_link(net_generic(net, ip6gre_net_id), nt); } + +unlock: + mutex_unlock(&ign->tunnels_lock); + return err; } @@ -2269,14 +2342,20 @@ static int ip6erspan_changelink(struct net_device *dev, struct nlattr *tb[], struct ip6_tnl *t = netdev_priv(dev); struct __ip6_tnl_parm p; struct ip6gre_net *ign; + int err = 0; if (!rtnl_dev_link_net_capable(dev, t->net)) return -EPERM; ign = net_generic(t->net, ip6gre_net_id); + + mutex_lock(&ign->tunnels_lock); + t = ip6gre_changelink_common(dev, tb, data, &p, extack); - if (IS_ERR(t)) - return PTR_ERR(t); + if (IS_ERR(t)) { + err = PTR_ERR(t); + goto unlock; + } ip6erspan_set_version(data, &p); ip6gre_tunnel_unlink_md(ign, t); @@ -2284,7 +2363,10 @@ static int ip6erspan_changelink(struct net_device *dev, struct nlattr *tb[], ip6erspan_tnl_change(t, &p, !tb[IFLA_MTU]); ip6erspan_tunnel_link_md(ign, t); ip6gre_tunnel_link(ign, t); - return 0; +unlock: + mutex_unlock(&ign->tunnels_lock); + + return err; } static struct rtnl_link_ops ip6gre_link_ops __read_mostly = { @@ -2311,6 +2393,7 @@ static struct rtnl_link_ops ip6gre_tap_ops __read_mostly = { .validate = ip6gre_tap_validate, .newlink = ip6gre_newlink, .changelink = ip6gre_changelink, + .dellink = ip6gre_dellink, .get_size = ip6gre_get_size, .fill_info = ip6gre_fill_info, .get_link_net = ip6_tnl_get_link_net, @@ -2325,6 +2408,7 @@ static struct rtnl_link_ops ip6erspan_tap_ops __read_mostly = { .validate = ip6erspan_tap_validate, .newlink = ip6erspan_newlink, .changelink = ip6erspan_changelink, + .dellink = ip6gre_dellink, .get_size = ip6gre_get_size, .fill_info = ip6gre_fill_info, .get_link_net = ip6_tnl_get_link_net,