net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <asm/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <linux/jhash.h>
  93 #include <net/dst.h>
  94 #include <net/net_namespace.h>
  95 #include <net/protocol.h>
  96 #include <net/ip.h>
  97 #include <net/route.h>
  98 #include <net/inetpeer.h>
  99 #include <net/sock.h>
 100 #include <net/ip_fib.h>
 101 #include <net/arp.h>
 102 #include <net/tcp.h>
 103 #include <net/icmp.h>
 104 #include <net/xfrm.h>
 105 #include <net/netevent.h>
 106 #include <net/rtnetlink.h>
 107 #ifdef CONFIG_SYSCTL
 108 #include <linux/sysctl.h>
 109 #include <linux/kmemleak.h>
 110 #endif
 111 #include <net/secure_seq.h>
 112
 113 #define RT_FL_TOS(oldflp4) \
 114         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 115
 116 #define IP_MAX_MTU      0xFFF0
 117
 118 #define RT_GC_TIMEOUT (300*HZ)
 119
 120 static int ip_rt_max_size;
 121 static int ip_rt_redirect_number __read_mostly  = 9;
 122 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 123 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 124 static int ip_rt_error_cost __read_mostly       = HZ;
 125 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 126 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 127 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 128 static int ip_rt_min_advmss __read_mostly       = 256;
 129
 130 /*
 131  *      Interface to generic destination cache.
 132  */
 133
 134 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 135 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 136 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 137 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 138 static void              ipv4_link_failure(struct sk_buff *skb);
 139 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 140                                            struct sk_buff *skb, u32 mtu);
 141 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 142                                         struct sk_buff *skb);
 143 static void             ipv4_dst_destroy(struct dst_entry *dst);
 144
 145 static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
 146                             int how)
 147 {
 148 }
 149
 150 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 151 {
 152         WARN_ON(1);
 153         return NULL;
 154 }
 155
 156 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 157                                            struct sk_buff *skb,
 158                                            const void *daddr);
 159
 160 static struct dst_ops ipv4_dst_ops = {
 161         .family =               AF_INET,
 162         .protocol =             cpu_to_be16(ETH_P_IP),
 163         .check =                ipv4_dst_check,
 164         .default_advmss =       ipv4_default_advmss,
 165         .mtu =                  ipv4_mtu,
 166         .cow_metrics =          ipv4_cow_metrics,
 167         .destroy =              ipv4_dst_destroy,
 168         .ifdown =               ipv4_dst_ifdown,
 169         .negative_advice =      ipv4_negative_advice,
 170         .link_failure =         ipv4_link_failure,
 171         .update_pmtu =          ip_rt_update_pmtu,
 172         .redirect =             ip_do_redirect,
 173         .local_out =            __ip_local_out,
 174         .neigh_lookup =         ipv4_neigh_lookup,
 175 };
 176
 177 #define ECN_OR_COST(class)      TC_PRIO_##class
 178
 179 const __u8 ip_tos2prio[16] = {
 180         TC_PRIO_BESTEFFORT,
 181         ECN_OR_COST(BESTEFFORT),
 182         TC_PRIO_BESTEFFORT,
 183         ECN_OR_COST(BESTEFFORT),
 184         TC_PRIO_BULK,
 185         ECN_OR_COST(BULK),
 186         TC_PRIO_BULK,
 187         ECN_OR_COST(BULK),
 188         TC_PRIO_INTERACTIVE,
 189         ECN_OR_COST(INTERACTIVE),
 190         TC_PRIO_INTERACTIVE,
 191         ECN_OR_COST(INTERACTIVE),
 192         TC_PRIO_INTERACTIVE_BULK,
 193         ECN_OR_COST(INTERACTIVE_BULK),
 194         TC_PRIO_INTERACTIVE_BULK,
 195         ECN_OR_COST(INTERACTIVE_BULK)
 196 };
 197 EXPORT_SYMBOL(ip_tos2prio);
 198
 199 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 200 #define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
 201
 202 #ifdef CONFIG_PROC_FS
 203 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 204 {
 205         if (*pos)
 206                 return NULL;
 207         return SEQ_START_TOKEN;
 208 }
 209
 210 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 211 {
 212         ++*pos;
 213         return NULL;
 214 }
 215
 216 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 217 {
 218 }
 219
 220 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 221 {
 222         if (v == SEQ_START_TOKEN)
 223                 seq_printf(seq, "%-127s\n",
 224                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 225                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 226                            "HHUptod\tSpecDst");
 227         return 0;
 228 }
 229
 230 static const struct seq_operations rt_cache_seq_ops = {
 231         .start  = rt_cache_seq_start,
 232         .next   = rt_cache_seq_next,
 233         .stop   = rt_cache_seq_stop,
 234         .show   = rt_cache_seq_show,
 235 };
 236
 237 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 238 {
 239         return seq_open(file, &rt_cache_seq_ops);
 240 }
 241
 242 static const struct file_operations rt_cache_seq_fops = {
 243         .owner   = THIS_MODULE,
 244         .open    = rt_cache_seq_open,
 245         .read    = seq_read,
 246         .llseek  = seq_lseek,
 247         .release = seq_release,
 248 };
 249
 250
 251 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 252 {
 253         int cpu;
 254
 255         if (*pos == 0)
 256                 return SEQ_START_TOKEN;
 257
 258         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 259                 if (!cpu_possible(cpu))
 260                         continue;
 261                 *pos = cpu+1;
 262                 return &per_cpu(rt_cache_stat, cpu);
 263         }
 264         return NULL;
 265 }
 266
 267 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 268 {
 269         int cpu;
 270
 271         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 272                 if (!cpu_possible(cpu))
 273                         continue;
 274                 *pos = cpu+1;
 275                 return &per_cpu(rt_cache_stat, cpu);
 276         }
 277         return NULL;
 278
 279 }
 280
 281 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 282 {
 283
 284 }
 285
 286 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 287 {
 288         struct rt_cache_stat *st = v;
 289
 290         if (v == SEQ_START_TOKEN) {
 291                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 292                 return 0;
 293         }
 294
 295         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 296                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 297                    dst_entries_get_slow(&ipv4_dst_ops),
 298                    st->in_hit,
 299                    st->in_slow_tot,
 300                    st->in_slow_mc,
 301                    st->in_no_route,
 302                    st->in_brd,
 303                    st->in_martian_dst,
 304                    st->in_martian_src,
 305
 306                    st->out_hit,
 307                    st->out_slow_tot,
 308                    st->out_slow_mc,
 309
 310                    st->gc_total,
 311                    st->gc_ignored,
 312                    st->gc_goal_miss,
 313                    st->gc_dst_overflow,
 314                    st->in_hlist_search,
 315                    st->out_hlist_search
 316                 );
 317         return 0;
 318 }
 319
 320 static const struct seq_operations rt_cpu_seq_ops = {
 321         .start  = rt_cpu_seq_start,
 322         .next   = rt_cpu_seq_next,
 323         .stop   = rt_cpu_seq_stop,
 324         .show   = rt_cpu_seq_show,
 325 };
 326
 327
 328 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 329 {
 330         return seq_open(file, &rt_cpu_seq_ops);
 331 }
 332
 333 static const struct file_operations rt_cpu_seq_fops = {
 334         .owner   = THIS_MODULE,
 335         .open    = rt_cpu_seq_open,
 336         .read    = seq_read,
 337         .llseek  = seq_lseek,
 338         .release = seq_release,
 339 };
 340
 341 #ifdef CONFIG_IP_ROUTE_CLASSID
 342 static int rt_acct_proc_show(struct seq_file *m, void *v)
 343 {
 344         struct ip_rt_acct *dst, *src;
 345         unsigned int i, j;
 346
 347         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 348         if (!dst)
 349                 return -ENOMEM;
 350
 351         for_each_possible_cpu(i) {
 352                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 353                 for (j = 0; j < 256; j++) {
 354                         dst[j].o_bytes   += src[j].o_bytes;
 355                         dst[j].o_packets += src[j].o_packets;
 356                         dst[j].i_bytes   += src[j].i_bytes;
 357                         dst[j].i_packets += src[j].i_packets;
 358                 }
 359         }
 360
 361         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 362         kfree(dst);
 363         return 0;
 364 }
 365
 366 static int rt_acct_proc_open(struct inode *inode, struct file *file)
 367 {
 368         return single_open(file, rt_acct_proc_show, NULL);
 369 }
 370
 371 static const struct file_operations rt_acct_proc_fops = {
 372         .owner          = THIS_MODULE,
 373         .open           = rt_acct_proc_open,
 374         .read           = seq_read,
 375         .llseek         = seq_lseek,
 376         .release        = single_release,
 377 };
 378 #endif
 379
 380 static int __net_init ip_rt_do_proc_init(struct net *net)
 381 {
 382         struct proc_dir_entry *pde;
 383
 384         pde = proc_create("rt_cache", S_IRUGO, net->proc_net,
 385                           &rt_cache_seq_fops);
 386         if (!pde)
 387                 goto err1;
 388
 389         pde = proc_create("rt_cache", S_IRUGO,
 390                           net->proc_net_stat, &rt_cpu_seq_fops);
 391         if (!pde)
 392                 goto err2;
 393
 394 #ifdef CONFIG_IP_ROUTE_CLASSID
 395         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
 396         if (!pde)
 397                 goto err3;
 398 #endif
 399         return 0;
 400
 401 #ifdef CONFIG_IP_ROUTE_CLASSID
 402 err3:
 403         remove_proc_entry("rt_cache", net->proc_net_stat);
 404 #endif
 405 err2:
 406         remove_proc_entry("rt_cache", net->proc_net);
 407 err1:
 408         return -ENOMEM;
 409 }
 410
 411 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 412 {
 413         remove_proc_entry("rt_cache", net->proc_net_stat);
 414         remove_proc_entry("rt_cache", net->proc_net);
 415 #ifdef CONFIG_IP_ROUTE_CLASSID
 416         remove_proc_entry("rt_acct", net->proc_net);
 417 #endif
 418 }
 419
 420 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 421         .init = ip_rt_do_proc_init,
 422         .exit = ip_rt_do_proc_exit,
 423 };
 424
 425 static int __init ip_rt_proc_init(void)
 426 {
 427         return register_pernet_subsys(&ip_rt_proc_ops);
 428 }
 429
 430 #else
 431 static inline int ip_rt_proc_init(void)
 432 {
 433         return 0;
 434 }
 435 #endif /* CONFIG_PROC_FS */
 436
 437 static inline bool rt_is_expired(const struct rtable *rth)
 438 {
 439         return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
 440 }
 441
 442 void rt_cache_flush(struct net *net)
 443 {
 444         rt_genid_bump(net);
 445 }
 446
 447 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 448                                            struct sk_buff *skb,
 449                                            const void *daddr)
 450 {
 451         struct net_device *dev = dst->dev;
 452         const __be32 *pkey = daddr;
 453         const struct rtable *rt;
 454         struct neighbour *n;
 455
 456         rt = (const struct rtable *) dst;
 457         if (rt->rt_gateway)
 458                 pkey = (const __be32 *) &rt->rt_gateway;
 459         else if (skb)
 460                 pkey = &ip_hdr(skb)->daddr;
 461
 462         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 463         if (n)
 464                 return n;
 465         return neigh_create(&arp_tbl, pkey, dev);
 466 }
 467
 468 atomic_t *ip_idents __read_mostly;
 469 EXPORT_SYMBOL(ip_idents);
 470
 471 void __ip_select_ident(struct iphdr *iph, int segs)
 472 {
 473         static u32 ip_idents_hashrnd __read_mostly;
 474         static bool hashrnd_initialized = false;
 475         u32 hash, id;
 476
 477         if (unlikely(!hashrnd_initialized)) {
 478                 hashrnd_initialized = true;
 479                 get_random_bytes(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
 480         }
 481
 482         hash = jhash_1word((__force u32)iph->daddr, ip_idents_hashrnd);
 483         id = ip_idents_reserve(hash, segs);
 484         iph->id = htons(id);
 485 }
 486 EXPORT_SYMBOL(__ip_select_ident);
 487
 488 static void __build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 489                              const struct iphdr *iph,
 490                              int oif, u8 tos,
 491                              u8 prot, u32 mark, int flow_flags)
 492 {
 493         if (sk) {
 494                 const struct inet_sock *inet = inet_sk(sk);
 495
 496                 oif = sk->sk_bound_dev_if;
 497                 mark = sk->sk_mark;
 498                 tos = RT_CONN_FLAGS(sk);
 499                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 500         }
 501         flowi4_init_output(fl4, oif, mark, tos,
 502                            RT_SCOPE_UNIVERSE, prot,
 503                            flow_flags,
 504                            iph->daddr, iph->saddr, 0, 0);
 505 }
 506
 507 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 508                                const struct sock *sk)
 509 {
 510         const struct iphdr *iph = ip_hdr(skb);
 511         int oif = skb->dev->ifindex;
 512         u8 tos = RT_TOS(iph->tos);
 513         u8 prot = iph->protocol;
 514         u32 mark = skb->mark;
 515
 516         __build_flow_key(fl4, sk, iph, oif, tos, prot, mark, 0);
 517 }
 518
 519 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 520 {
 521         const struct inet_sock *inet = inet_sk(sk);
 522         const struct ip_options_rcu *inet_opt;
 523         __be32 daddr = inet->inet_daddr;
 524
 525         rcu_read_lock();
 526         inet_opt = rcu_dereference(inet->inet_opt);
 527         if (inet_opt && inet_opt->opt.srr)
 528                 daddr = inet_opt->opt.faddr;
 529         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 530                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 531                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 532                            inet_sk_flowi_flags(sk),
 533                            daddr, inet->inet_saddr, 0, 0);
 534         rcu_read_unlock();
 535 }
 536
 537 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 538                                  const struct sk_buff *skb)
 539 {
 540         if (skb)
 541                 build_skb_flow_key(fl4, skb, sk);
 542         else
 543                 build_sk_flow_key(fl4, sk);
 544 }
 545
 546 static inline void rt_free(struct rtable *rt)
 547 {
 548         call_rcu(&rt->dst.rcu_head, dst_rcu_free);
 549 }
 550
 551 static DEFINE_SPINLOCK(fnhe_lock);
 552
 553 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 554 {
 555         struct fib_nh_exception *fnhe, *oldest;
 556         struct rtable *orig;
 557
 558         oldest = rcu_dereference(hash->chain);
 559         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 560              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 561                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 562                         oldest = fnhe;
 563         }
 564         orig = rcu_dereference(oldest->fnhe_rth);
 565         if (orig) {
 566                 RCU_INIT_POINTER(oldest->fnhe_rth, NULL);
 567                 rt_free(orig);
 568         }
 569         return oldest;
 570 }
 571
 572 static inline u32 fnhe_hashfun(__be32 daddr)
 573 {
 574         u32 hval;
 575
 576         hval = (__force u32) daddr;
 577         hval ^= (hval >> 11) ^ (hval >> 22);
 578
 579         return hval & (FNHE_HASH_SIZE - 1);
 580 }
 581
 582 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 583                                   u32 pmtu, unsigned long expires)
 584 {
 585         struct fnhe_hash_bucket *hash;
 586         struct fib_nh_exception *fnhe;
 587         int depth;
 588         u32 hval = fnhe_hashfun(daddr);
 589
 590         spin_lock_bh(&fnhe_lock);
 591
 592         hash = nh->nh_exceptions;
 593         if (!hash) {
 594                 hash = kzalloc(FNHE_HASH_SIZE * sizeof(*hash), GFP_ATOMIC);
 595                 if (!hash)
 596                         goto out_unlock;
 597                 nh->nh_exceptions = hash;
 598         }
 599
 600         hash += hval;
 601
 602         depth = 0;
 603         for (fnhe = rcu_dereference(hash->chain); fnhe;
 604              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 605                 if (fnhe->fnhe_daddr == daddr)
 606                         break;
 607                 depth++;
 608         }
 609
 610         if (fnhe) {
 611                 if (gw)
 612                         fnhe->fnhe_gw = gw;
 613                 if (pmtu) {
 614                         fnhe->fnhe_pmtu = pmtu;
 615                         fnhe->fnhe_expires = expires;
 616                 }
 617         } else {
 618                 if (depth > FNHE_RECLAIM_DEPTH)
 619                         fnhe = fnhe_oldest(hash);
 620                 else {
 621                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 622                         if (!fnhe)
 623                                 goto out_unlock;
 624
 625                         fnhe->fnhe_next = hash->chain;
 626                         rcu_assign_pointer(hash->chain, fnhe);
 627                 }
 628                 fnhe->fnhe_daddr = daddr;
 629                 fnhe->fnhe_gw = gw;
 630                 fnhe->fnhe_pmtu = pmtu;
 631                 fnhe->fnhe_expires = expires;
 632         }
 633
 634         fnhe->fnhe_stamp = jiffies;
 635
 636 out_unlock:
 637         spin_unlock_bh(&fnhe_lock);
 638         return;
 639 }
 640
 641 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 642                              bool kill_route)
 643 {
 644         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 645         __be32 old_gw = ip_hdr(skb)->saddr;
 646         struct net_device *dev = skb->dev;
 647         struct in_device *in_dev;
 648         struct fib_result res;
 649         struct neighbour *n;
 650         struct net *net;
 651
 652         switch (icmp_hdr(skb)->code & 7) {
 653         case ICMP_REDIR_NET:
 654         case ICMP_REDIR_NETTOS:
 655         case ICMP_REDIR_HOST:
 656         case ICMP_REDIR_HOSTTOS:
 657                 break;
 658
 659         default:
 660                 return;
 661         }
 662
 663         if (rt->rt_gateway != old_gw)
 664                 return;
 665
 666         in_dev = __in_dev_get_rcu(dev);
 667         if (!in_dev)
 668                 return;
 669
 670         net = dev_net(dev);
 671         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 672             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 673             ipv4_is_zeronet(new_gw))
 674                 goto reject_redirect;
 675
 676         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 677                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 678                         goto reject_redirect;
 679                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 680                         goto reject_redirect;
 681         } else {
 682                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 683                         goto reject_redirect;
 684         }
 685
 686         n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw);
 687         if (n) {
 688                 if (!(n->nud_state & NUD_VALID)) {
 689                         neigh_event_send(n, NULL);
 690                 } else {
 691                         if (fib_lookup(net, fl4, &res) == 0) {
 692                                 struct fib_nh *nh = &FIB_RES_NH(res);
 693
 694                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 695                                                       0, 0);
 696                         }
 697                         if (kill_route)
 698                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 699                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 700                 }
 701                 neigh_release(n);
 702         }
 703         return;
 704
 705 reject_redirect:
 706 #ifdef CONFIG_IP_ROUTE_VERBOSE
 707         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 708                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 709                 __be32 daddr = iph->daddr;
 710                 __be32 saddr = iph->saddr;
 711
 712                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 713                                      "  Advised path = %pI4 -> %pI4\n",
 714                                      &old_gw, dev->name, &new_gw,
 715                                      &saddr, &daddr);
 716         }
 717 #endif
 718         ;
 719 }
 720
 721 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 722 {
 723         struct rtable *rt;
 724         struct flowi4 fl4;
 725         const struct iphdr *iph = (const struct iphdr *) skb->data;
 726         int oif = skb->dev->ifindex;
 727         u8 tos = RT_TOS(iph->tos);
 728         u8 prot = iph->protocol;
 729         u32 mark = skb->mark;
 730
 731         rt = (struct rtable *) dst;
 732
 733         __build_flow_key(&fl4, sk, iph, oif, tos, prot, mark, 0);
 734         __ip_do_redirect(rt, skb, &fl4, true);
 735 }
 736
 737 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 738 {
 739         struct rtable *rt = (struct rtable *)dst;
 740         struct dst_entry *ret = dst;
 741
 742         if (rt) {
 743                 if (dst->obsolete > 0) {
 744                         ip_rt_put(rt);
 745                         ret = NULL;
 746                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 747                            rt->dst.expires) {
 748                         ip_rt_put(rt);
 749                         ret = NULL;
 750                 }
 751         }
 752         return ret;
 753 }
 754
 755 /*
 756  * Algorithm:
 757  *      1. The first ip_rt_redirect_number redirects are sent
 758  *         with exponential backoff, then we stop sending them at all,
 759  *         assuming that the host ignores our redirects.
 760  *      2. If we did not see packets requiring redirects
 761  *         during ip_rt_redirect_silence, we assume that the host
 762  *         forgot redirected route and start to send redirects again.
 763  *
 764  * This algorithm is much cheaper and more intelligent than dumb load limiting
 765  * in icmp.c.
 766  *
 767  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 768  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 769  */
 770
 771 void ip_rt_send_redirect(struct sk_buff *skb)
 772 {
 773         struct rtable *rt = skb_rtable(skb);
 774         struct in_device *in_dev;
 775         struct inet_peer *peer;
 776         struct net *net;
 777         int log_martians;
 778
 779         rcu_read_lock();
 780         in_dev = __in_dev_get_rcu(rt->dst.dev);
 781         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 782                 rcu_read_unlock();
 783                 return;
 784         }
 785         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 786         rcu_read_unlock();
 787
 788         net = dev_net(rt->dst.dev);
 789         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
 790         if (!peer) {
 791                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 792                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 793                 return;
 794         }
 795
 796         /* No redirected packets during ip_rt_redirect_silence;
 797          * reset the algorithm.
 798          */
 799         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
 800                 peer->rate_tokens = 0;
 801
 802         /* Too many ignored redirects; do not send anything
 803          * set dst.rate_last to the last seen redirected packet.
 804          */
 805         if (peer->rate_tokens >= ip_rt_redirect_number) {
 806                 peer->rate_last = jiffies;
 807                 goto out_put_peer;
 808         }
 809
 810         /* Check for load limit; set rate_last to the latest sent
 811          * redirect.
 812          */
 813         if (peer->rate_tokens == 0 ||
 814             time_after(jiffies,
 815                        (peer->rate_last +
 816                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 817                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 818
 819                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 820                 peer->rate_last = jiffies;
 821                 ++peer->rate_tokens;
 822 #ifdef CONFIG_IP_ROUTE_VERBOSE
 823                 if (log_martians &&
 824                     peer->rate_tokens == ip_rt_redirect_number)
 825                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 826                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 827                                              &ip_hdr(skb)->daddr, &gw);
 828 #endif
 829         }
 830 out_put_peer:
 831         inet_putpeer(peer);
 832 }
 833
 834 static int ip_error(struct sk_buff *skb)
 835 {
 836         struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
 837         struct rtable *rt = skb_rtable(skb);
 838         struct inet_peer *peer;
 839         unsigned long now;
 840         struct net *net;
 841         bool send;
 842         int code;
 843
 844         net = dev_net(rt->dst.dev);
 845         if (!IN_DEV_FORWARD(in_dev)) {
 846                 switch (rt->dst.error) {
 847                 case EHOSTUNREACH:
 848                         IP_INC_STATS_BH(net, IPSTATS_MIB_INADDRERRORS);
 849                         break;
 850
 851                 case ENETUNREACH:
 852                         IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
 853                         break;
 854                 }
 855                 goto out;
 856         }
 857
 858         switch (rt->dst.error) {
 859         case EINVAL:
 860         default:
 861                 goto out;
 862         case EHOSTUNREACH:
 863                 code = ICMP_HOST_UNREACH;
 864                 break;
 865         case ENETUNREACH:
 866                 code = ICMP_NET_UNREACH;
 867                 IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
 868                 break;
 869         case EACCES:
 870                 code = ICMP_PKT_FILTERED;
 871                 break;
 872         }
 873
 874         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
 875
 876         send = true;
 877         if (peer) {
 878                 now = jiffies;
 879                 peer->rate_tokens += now - peer->rate_last;
 880                 if (peer->rate_tokens > ip_rt_error_burst)
 881                         peer->rate_tokens = ip_rt_error_burst;
 882                 peer->rate_last = now;
 883                 if (peer->rate_tokens >= ip_rt_error_cost)
 884                         peer->rate_tokens -= ip_rt_error_cost;
 885                 else
 886                         send = false;
 887                 inet_putpeer(peer);
 888         }
 889         if (send)
 890                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 891
 892 out:    kfree_skb(skb);
 893         return 0;
 894 }
 895
 896 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
 897 {
 898         struct dst_entry *dst = &rt->dst;
 899         struct fib_result res;
 900
 901         if (dst_metric_locked(dst, RTAX_MTU))
 902                 return;
 903
 904         if (dst->dev->mtu < mtu)
 905                 return;
 906
 907         if (mtu < ip_rt_min_pmtu)
 908                 mtu = ip_rt_min_pmtu;
 909
 910         if (!rt->rt_pmtu) {
 911                 dst->obsolete = DST_OBSOLETE_KILL;
 912         } else {
 913                 rt->rt_pmtu = mtu;
 914                 dst->expires = max(1UL, jiffies + ip_rt_mtu_expires);
 915         }
 916
 917         rcu_read_lock();
 918         if (fib_lookup(dev_net(dst->dev), fl4, &res) == 0) {
 919                 struct fib_nh *nh = &FIB_RES_NH(res);
 920
 921                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu,
 922                                       jiffies + ip_rt_mtu_expires);
 923         }
 924         rcu_read_unlock();
 925 }
 926
 927 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 928                               struct sk_buff *skb, u32 mtu)
 929 {
 930         struct rtable *rt = (struct rtable *) dst;
 931         struct flowi4 fl4;
 932
 933         ip_rt_build_flow_key(&fl4, sk, skb);
 934         __ip_rt_update_pmtu(rt, &fl4, mtu);
 935 }
 936
 937 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
 938                       int oif, u32 mark, u8 protocol, int flow_flags)
 939 {
 940         const struct iphdr *iph = (const struct iphdr *) skb->data;
 941         struct flowi4 fl4;
 942         struct rtable *rt;
 943
 944         __build_flow_key(&fl4, NULL, iph, oif,
 945                          RT_TOS(iph->tos), protocol, mark, flow_flags);
 946         rt = __ip_route_output_key(net, &fl4);
 947         if (!IS_ERR(rt)) {
 948                 __ip_rt_update_pmtu(rt, &fl4, mtu);
 949                 ip_rt_put(rt);
 950         }
 951 }
 952 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
 953
 954 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
 955 {
 956         const struct iphdr *iph = (const struct iphdr *) skb->data;
 957         struct flowi4 fl4;
 958         struct rtable *rt;
 959
 960         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
 961         rt = __ip_route_output_key(sock_net(sk), &fl4);
 962         if (!IS_ERR(rt)) {
 963                 __ip_rt_update_pmtu(rt, &fl4, mtu);
 964                 ip_rt_put(rt);
 965         }
 966 }
 967
 968 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
 969 {
 970         const struct iphdr *iph = (const struct iphdr *) skb->data;
 971         struct flowi4 fl4;
 972         struct rtable *rt;
 973         struct dst_entry *odst = NULL;
 974         bool new = false;
 975
 976         bh_lock_sock(sk);
 977         odst = sk_dst_get(sk);
 978
 979         if (sock_owned_by_user(sk) || !odst) {
 980                 __ipv4_sk_update_pmtu(skb, sk, mtu);
 981                 goto out;
 982         }
 983
 984         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
 985
 986         rt = (struct rtable *)odst;
 987         if (odst->obsolete && odst->ops->check(odst, 0) == NULL) {
 988                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
 989                 if (IS_ERR(rt))
 990                         goto out;
 991
 992                 new = true;
 993         }
 994
 995         __ip_rt_update_pmtu((struct rtable *) rt->dst.path, &fl4, mtu);
 996
 997         if (!dst_check(&rt->dst, 0)) {
 998                 if (new)
 999                         dst_release(&rt->dst);
1000
1001                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1002                 if (IS_ERR(rt))
1003                         goto out;
1004
1005                 new = true;
1006         }
1007
1008         if (new)
1009                 sk_dst_set(sk, &rt->dst);
1010
1011 out:
1012         bh_unlock_sock(sk);
1013         dst_release(odst);
1014 }
1015 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1016
1017 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1018                    int oif, u32 mark, u8 protocol, int flow_flags)
1019 {
1020         const struct iphdr *iph = (const struct iphdr *) skb->data;
1021         struct flowi4 fl4;
1022         struct rtable *rt;
1023
1024         __build_flow_key(&fl4, NULL, iph, oif,
1025                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1026         rt = __ip_route_output_key(net, &fl4);
1027         if (!IS_ERR(rt)) {
1028                 __ip_do_redirect(rt, skb, &fl4, false);
1029                 ip_rt_put(rt);
1030         }
1031 }
1032 EXPORT_SYMBOL_GPL(ipv4_redirect);
1033
1034 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1035 {
1036         const struct iphdr *iph = (const struct iphdr *) skb->data;
1037         struct flowi4 fl4;
1038         struct rtable *rt;
1039
1040         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1041         rt = __ip_route_output_key(sock_net(sk), &fl4);
1042         if (!IS_ERR(rt)) {
1043                 __ip_do_redirect(rt, skb, &fl4, false);
1044                 ip_rt_put(rt);
1045         }
1046 }
1047 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1048
1049 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1050 {
1051         struct rtable *rt = (struct rtable *) dst;
1052
1053         /* All IPV4 dsts are created with ->obsolete set to the value
1054          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1055          * into this function always.
1056          *
1057          * When a PMTU/redirect information update invalidates a
1058          * route, this is indicated by setting obsolete to
1059          * DST_OBSOLETE_KILL.
1060          */
1061         if (dst->obsolete == DST_OBSOLETE_KILL || rt_is_expired(rt))
1062                 return NULL;
1063         return dst;
1064 }
1065
1066 static void ipv4_link_failure(struct sk_buff *skb)
1067 {
1068         struct rtable *rt;
1069
1070         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1071
1072         rt = skb_rtable(skb);
1073         if (rt)
1074                 dst_set_expires(&rt->dst, 0);
1075 }
1076
1077 static int ip_rt_bug(struct sk_buff *skb)
1078 {
1079         pr_debug("%s: %pI4 -> %pI4, %s\n",
1080                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1081                  skb->dev ? skb->dev->name : "?");
1082         kfree_skb(skb);
1083         WARN_ON(1);
1084         return 0;
1085 }
1086
1087 /*
1088    We do not cache source address of outgoing interface,
1089    because it is used only by IP RR, TS and SRR options,
1090    so that it out of fast path.
1091
1092    BTW remember: "addr" is allowed to be not aligned
1093    in IP options!
1094  */
1095
1096 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1097 {
1098         __be32 src;
1099
1100         if (rt_is_output_route(rt))
1101                 src = ip_hdr(skb)->saddr;
1102         else {
1103                 struct fib_result res;
1104                 struct flowi4 fl4;
1105                 struct iphdr *iph;
1106
1107                 iph = ip_hdr(skb);
1108
1109                 memset(&fl4, 0, sizeof(fl4));
1110                 fl4.daddr = iph->daddr;
1111                 fl4.saddr = iph->saddr;
1112                 fl4.flowi4_tos = RT_TOS(iph->tos);
1113                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1114                 fl4.flowi4_iif = skb->dev->ifindex;
1115                 fl4.flowi4_mark = skb->mark;
1116
1117                 rcu_read_lock();
1118                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res) == 0)
1119                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1120                 else
1121                         src = inet_select_addr(rt->dst.dev,
1122                                                rt_nexthop(rt, iph->daddr),
1123                                                RT_SCOPE_UNIVERSE);
1124                 rcu_read_unlock();
1125         }
1126         memcpy(addr, &src, 4);
1127 }
1128
1129 #ifdef CONFIG_IP_ROUTE_CLASSID
1130 static void set_class_tag(struct rtable *rt, u32 tag)
1131 {
1132         if (!(rt->dst.tclassid & 0xFFFF))
1133                 rt->dst.tclassid |= tag & 0xFFFF;
1134         if (!(rt->dst.tclassid & 0xFFFF0000))
1135                 rt->dst.tclassid |= tag & 0xFFFF0000;
1136 }
1137 #endif
1138
1139 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1140 {
1141         unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1142
1143         if (advmss == 0) {
1144                 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1145                                ip_rt_min_advmss);
1146                 if (advmss > 65535 - 40)
1147                         advmss = 65535 - 40;
1148         }
1149         return advmss;
1150 }
1151
1152 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1153 {
1154         const struct rtable *rt = (const struct rtable *) dst;
1155         unsigned int mtu = rt->rt_pmtu;
1156
1157         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1158                 mtu = dst_metric_raw(dst, RTAX_MTU);
1159
1160         if (mtu)
1161                 return mtu;
1162
1163         mtu = dst->dev->mtu;
1164
1165         if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1166                 if (rt->rt_uses_gateway && mtu > 576)
1167                         mtu = 576;
1168         }
1169
1170         if (mtu > IP_MAX_MTU)
1171                 mtu = IP_MAX_MTU;
1172
1173         return mtu;
1174 }
1175
1176 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1177 {
1178         struct fnhe_hash_bucket *hash = nh->nh_exceptions;
1179         struct fib_nh_exception *fnhe;
1180         u32 hval;
1181
1182         if (!hash)
1183                 return NULL;
1184
1185         hval = fnhe_hashfun(daddr);
1186
1187         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1188              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1189                 if (fnhe->fnhe_daddr == daddr)
1190                         return fnhe;
1191         }
1192         return NULL;
1193 }
1194
1195 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1196                               __be32 daddr)
1197 {
1198         bool ret = false;
1199
1200         spin_lock_bh(&fnhe_lock);
1201
1202         if (daddr == fnhe->fnhe_daddr) {
1203                 struct rtable *orig = rcu_dereference(fnhe->fnhe_rth);
1204                 if (orig && rt_is_expired(orig)) {
1205                         fnhe->fnhe_gw = 0;
1206                         fnhe->fnhe_pmtu = 0;
1207                         fnhe->fnhe_expires = 0;
1208                 }
1209                 if (fnhe->fnhe_pmtu) {
1210                         unsigned long expires = fnhe->fnhe_expires;
1211                         unsigned long diff = expires - jiffies;
1212
1213                         if (time_before(jiffies, expires)) {
1214                                 rt->rt_pmtu = fnhe->fnhe_pmtu;
1215                                 dst_set_expires(&rt->dst, diff);
1216                         }
1217                 }
1218                 if (fnhe->fnhe_gw) {
1219                         rt->rt_flags |= RTCF_REDIRECTED;
1220                         rt->rt_gateway = fnhe->fnhe_gw;
1221                         rt->rt_uses_gateway = 1;
1222                 } else if (!rt->rt_gateway)
1223                         rt->rt_gateway = daddr;
1224
1225                 rcu_assign_pointer(fnhe->fnhe_rth, rt);
1226                 if (orig)
1227                         rt_free(orig);
1228
1229                 fnhe->fnhe_stamp = jiffies;
1230                 ret = true;
1231         }
1232         spin_unlock_bh(&fnhe_lock);
1233
1234         return ret;
1235 }
1236
1237 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1238 {
1239         struct rtable *orig, *prev, **p;
1240         bool ret = true;
1241
1242         if (rt_is_input_route(rt)) {
1243                 p = (struct rtable **)&nh->nh_rth_input;
1244         } else {
1245                 p = (struct rtable **)__this_cpu_ptr(nh->nh_pcpu_rth_output);
1246         }
1247         orig = *p;
1248
1249         prev = cmpxchg(p, orig, rt);
1250         if (prev == orig) {
1251                 if (orig)
1252                         rt_free(orig);
1253         } else
1254                 ret = false;
1255
1256         return ret;
1257 }
1258
1259 static DEFINE_SPINLOCK(rt_uncached_lock);
1260 static LIST_HEAD(rt_uncached_list);
1261
1262 static void rt_add_uncached_list(struct rtable *rt)
1263 {
1264         spin_lock_bh(&rt_uncached_lock);
1265         list_add_tail(&rt->rt_uncached, &rt_uncached_list);
1266         spin_unlock_bh(&rt_uncached_lock);
1267 }
1268
1269 static void ipv4_dst_destroy(struct dst_entry *dst)
1270 {
1271         struct rtable *rt = (struct rtable *) dst;
1272
1273         if (!list_empty(&rt->rt_uncached)) {
1274                 spin_lock_bh(&rt_uncached_lock);
1275                 list_del(&rt->rt_uncached);
1276                 spin_unlock_bh(&rt_uncached_lock);
1277         }
1278 }
1279
1280 void rt_flush_dev(struct net_device *dev)
1281 {
1282         if (!list_empty(&rt_uncached_list)) {
1283                 struct net *net = dev_net(dev);
1284                 struct rtable *rt;
1285
1286                 spin_lock_bh(&rt_uncached_lock);
1287                 list_for_each_entry(rt, &rt_uncached_list, rt_uncached) {
1288                         if (rt->dst.dev != dev)
1289                                 continue;
1290                         rt->dst.dev = net->loopback_dev;
1291                         dev_hold(rt->dst.dev);
1292                         dev_put(dev);
1293                 }
1294                 spin_unlock_bh(&rt_uncached_lock);
1295         }
1296 }
1297
1298 static bool rt_cache_valid(const struct rtable *rt)
1299 {
1300         return  rt &&
1301                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1302                 !rt_is_expired(rt);
1303 }
1304
1305 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1306                            const struct fib_result *res,
1307                            struct fib_nh_exception *fnhe,
1308                            struct fib_info *fi, u16 type, u32 itag)
1309 {
1310         bool cached = false;
1311
1312         if (fi) {
1313                 struct fib_nh *nh = &FIB_RES_NH(*res);
1314
1315                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1316                         rt->rt_gateway = nh->nh_gw;
1317                         rt->rt_uses_gateway = 1;
1318                 }
1319                 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1320 #ifdef CONFIG_IP_ROUTE_CLASSID
1321                 rt->dst.tclassid = nh->nh_tclassid;
1322 #endif
1323                 if (unlikely(fnhe))
1324                         cached = rt_bind_exception(rt, fnhe, daddr);
1325                 else if (!(rt->dst.flags & DST_NOCACHE))
1326                         cached = rt_cache_route(nh, rt);
1327                 if (unlikely(!cached)) {
1328                         /* Routes we intend to cache in nexthop exception or
1329                          * FIB nexthop have the DST_NOCACHE bit clear.
1330                          * However, if we are unsuccessful at storing this
1331                          * route into the cache we really need to set it.
1332                          */
1333                         rt->dst.flags |= DST_NOCACHE;
1334                         if (!rt->rt_gateway)
1335                                 rt->rt_gateway = daddr;
1336                         rt_add_uncached_list(rt);
1337                 }
1338         } else
1339                 rt_add_uncached_list(rt);
1340
1341 #ifdef CONFIG_IP_ROUTE_CLASSID
1342 #ifdef CONFIG_IP_MULTIPLE_TABLES
1343         set_class_tag(rt, res->tclassid);
1344 #endif
1345         set_class_tag(rt, itag);
1346 #endif
1347 }
1348
1349 static struct rtable *rt_dst_alloc(struct net_device *dev,
1350                                    bool nopolicy, bool noxfrm, bool will_cache)
1351 {
1352         return dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1353                          (will_cache ? 0 : (DST_HOST | DST_NOCACHE)) |
1354                          (nopolicy ? DST_NOPOLICY : 0) |
1355                          (noxfrm ? DST_NOXFRM : 0));
1356 }
1357
1358 /* called in rcu_read_lock() section */
1359 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1360                                 u8 tos, struct net_device *dev, int our)
1361 {
1362         struct rtable *rth;
1363         struct in_device *in_dev = __in_dev_get_rcu(dev);
1364         u32 itag = 0;
1365         int err;
1366
1367         /* Primary sanity checks. */
1368
1369         if (in_dev == NULL)
1370                 return -EINVAL;
1371
1372         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1373             skb->protocol != htons(ETH_P_IP))
1374                 goto e_inval;
1375
1376         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1377                 if (ipv4_is_loopback(saddr))
1378                         goto e_inval;
1379
1380         if (ipv4_is_zeronet(saddr)) {
1381                 if (!ipv4_is_local_multicast(daddr))
1382                         goto e_inval;
1383         } else {
1384                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1385                                           in_dev, &itag);
1386                 if (err < 0)
1387                         goto e_err;
1388         }
1389         rth = rt_dst_alloc(dev_net(dev)->loopback_dev,
1390                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1391         if (!rth)
1392                 goto e_nobufs;
1393
1394 #ifdef CONFIG_IP_ROUTE_CLASSID
1395         rth->dst.tclassid = itag;
1396 #endif
1397         rth->dst.output = ip_rt_bug;
1398
1399         rth->rt_genid   = rt_genid(dev_net(dev));
1400         rth->rt_flags   = RTCF_MULTICAST;
1401         rth->rt_type    = RTN_MULTICAST;
1402         rth->rt_is_input= 1;
1403         rth->rt_iif     = 0;
1404         rth->rt_pmtu    = 0;
1405         rth->rt_gateway = 0;
1406         rth->rt_uses_gateway = 0;
1407         INIT_LIST_HEAD(&rth->rt_uncached);
1408         if (our) {
1409                 rth->dst.input= ip_local_deliver;
1410                 rth->rt_flags |= RTCF_LOCAL;
1411         }
1412
1413 #ifdef CONFIG_IP_MROUTE
1414         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1415                 rth->dst.input = ip_mr_input;
1416 #endif
1417         RT_CACHE_STAT_INC(in_slow_mc);
1418
1419         skb_dst_set(skb, &rth->dst);
1420         return 0;
1421
1422 e_nobufs:
1423         return -ENOBUFS;
1424 e_inval:
1425         return -EINVAL;
1426 e_err:
1427         return err;
1428 }
1429
1430
1431 static void ip_handle_martian_source(struct net_device *dev,
1432                                      struct in_device *in_dev,
1433                                      struct sk_buff *skb,
1434                                      __be32 daddr,
1435                                      __be32 saddr)
1436 {
1437         RT_CACHE_STAT_INC(in_martian_src);
1438 #ifdef CONFIG_IP_ROUTE_VERBOSE
1439         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1440                 /*
1441                  *      RFC1812 recommendation, if source is martian,
1442                  *      the only hint is MAC header.
1443                  */
1444                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1445                         &daddr, &saddr, dev->name);
1446                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1447                         print_hex_dump(KERN_WARNING, "ll header: ",
1448                                        DUMP_PREFIX_OFFSET, 16, 1,
1449                                        skb_mac_header(skb),
1450                                        dev->hard_header_len, true);
1451                 }
1452         }
1453 #endif
1454 }
1455
1456 /* called in rcu_read_lock() section */
1457 static int __mkroute_input(struct sk_buff *skb,
1458                            const struct fib_result *res,
1459                            struct in_device *in_dev,
1460                            __be32 daddr, __be32 saddr, u32 tos)
1461 {
1462         struct rtable *rth;
1463         int err;
1464         struct in_device *out_dev;
1465         unsigned int flags = 0;
1466         bool do_cache;
1467         u32 itag = 0;
1468
1469         /* get a working reference to the output device */
1470         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1471         if (out_dev == NULL) {
1472                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1473                 return -EINVAL;
1474         }
1475
1476         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1477                                   in_dev->dev, in_dev, &itag);
1478         if (err < 0) {
1479                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1480                                          saddr);
1481
1482                 goto cleanup;
1483         }
1484
1485         do_cache = res->fi && !itag;
1486         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1487             (IN_DEV_SHARED_MEDIA(out_dev) ||
1488              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res)))) {
1489                 flags |= RTCF_DOREDIRECT;
1490                 do_cache = false;
1491         }
1492
1493         if (skb->protocol != htons(ETH_P_IP)) {
1494                 /* Not IP (i.e. ARP). Do not create route, if it is
1495                  * invalid for proxy arp. DNAT routes are always valid.
1496                  *
1497                  * Proxy arp feature have been extended to allow, ARP
1498                  * replies back to the same interface, to support
1499                  * Private VLAN switch technologies. See arp.c.
1500                  */
1501                 if (out_dev == in_dev &&
1502                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1503                         err = -EINVAL;
1504                         goto cleanup;
1505                 }
1506         }
1507
1508         if (do_cache) {
1509                 rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1510                 if (rt_cache_valid(rth)) {
1511                         skb_dst_set_noref(skb, &rth->dst);
1512                         goto out;
1513                 }
1514         }
1515
1516         rth = rt_dst_alloc(out_dev->dev,
1517                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1518                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1519         if (!rth) {
1520                 err = -ENOBUFS;
1521                 goto cleanup;
1522         }
1523
1524         rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
1525         rth->rt_flags = flags;
1526         rth->rt_type = res->type;
1527         rth->rt_is_input = 1;
1528         rth->rt_iif     = 0;
1529         rth->rt_pmtu    = 0;
1530         rth->rt_gateway = 0;
1531         rth->rt_uses_gateway = 0;
1532         INIT_LIST_HEAD(&rth->rt_uncached);
1533         RT_CACHE_STAT_INC(in_slow_tot);
1534
1535         rth->dst.input = ip_forward;
1536         rth->dst.output = ip_output;
1537
1538         rt_set_nexthop(rth, daddr, res, NULL, res->fi, res->type, itag);
1539         skb_dst_set(skb, &rth->dst);
1540 out:
1541         err = 0;
1542  cleanup:
1543         return err;
1544 }
1545
1546 static int ip_mkroute_input(struct sk_buff *skb,
1547                             struct fib_result *res,
1548                             const struct flowi4 *fl4,
1549                             struct in_device *in_dev,
1550                             __be32 daddr, __be32 saddr, u32 tos)
1551 {
1552 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1553         if (res->fi && res->fi->fib_nhs > 1)
1554                 fib_select_multipath(res);
1555 #endif
1556
1557         /* create a routing cache entry */
1558         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1559 }
1560
1561 /*
1562  *      NOTE. We drop all the packets that has local source
1563  *      addresses, because every properly looped back packet
1564  *      must have correct destination already attached by output routine.
1565  *
1566  *      Such approach solves two big problems:
1567  *      1. Not simplex devices are handled properly.
1568  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1569  *      called with rcu_read_lock()
1570  */
1571
1572 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1573                                u8 tos, struct net_device *dev)
1574 {
1575         struct fib_result res;
1576         struct in_device *in_dev = __in_dev_get_rcu(dev);
1577         struct flowi4   fl4;
1578         unsigned int    flags = 0;
1579         u32             itag = 0;
1580         struct rtable   *rth;
1581         int             err = -EINVAL;
1582         struct net    *net = dev_net(dev);
1583         bool do_cache;
1584
1585         /* IP on this device is disabled. */
1586
1587         if (!in_dev)
1588                 goto out;
1589
1590         /* Check for the most weird martians, which can be not detected
1591            by fib_lookup.
1592          */
1593
1594         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1595                 goto martian_source;
1596
1597         res.fi = NULL;
1598         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1599                 goto brd_input;
1600
1601         /* Accept zero addresses only to limited broadcast;
1602          * I even do not know to fix it or not. Waiting for complains :-)
1603          */
1604         if (ipv4_is_zeronet(saddr))
1605                 goto martian_source;
1606
1607         if (ipv4_is_zeronet(daddr))
1608                 goto martian_destination;
1609
1610         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1611          * and call it once if daddr or/and saddr are loopback addresses
1612          */
1613         if (ipv4_is_loopback(daddr)) {
1614                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1615                         goto martian_destination;
1616         } else if (ipv4_is_loopback(saddr)) {
1617                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1618                         goto martian_source;
1619         }
1620
1621         /*
1622          *      Now we are ready to route packet.
1623          */
1624         fl4.flowi4_oif = 0;
1625         fl4.flowi4_iif = dev->ifindex;
1626         fl4.flowi4_mark = skb->mark;
1627         fl4.flowi4_tos = tos;
1628         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1629         fl4.daddr = daddr;
1630         fl4.saddr = saddr;
1631         err = fib_lookup(net, &fl4, &res);
1632         if (err != 0)
1633                 goto no_route;
1634
1635         if (res.type == RTN_BROADCAST)
1636                 goto brd_input;
1637
1638         if (res.type == RTN_LOCAL) {
1639                 err = fib_validate_source(skb, saddr, daddr, tos,
1640                                           LOOPBACK_IFINDEX,
1641                                           dev, in_dev, &itag);
1642                 if (err < 0)
1643                         goto martian_source_keep_err;
1644                 goto local_input;
1645         }
1646
1647         if (!IN_DEV_FORWARD(in_dev))
1648                 goto no_route;
1649         if (res.type != RTN_UNICAST)
1650                 goto martian_destination;
1651
1652         err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
1653 out:    return err;
1654
1655 brd_input:
1656         if (skb->protocol != htons(ETH_P_IP))
1657                 goto e_inval;
1658
1659         if (!ipv4_is_zeronet(saddr)) {
1660                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1661                                           in_dev, &itag);
1662                 if (err < 0)
1663                         goto martian_source_keep_err;
1664         }
1665         flags |= RTCF_BROADCAST;
1666         res.type = RTN_BROADCAST;
1667         RT_CACHE_STAT_INC(in_brd);
1668
1669 local_input:
1670         do_cache = false;
1671         if (res.fi) {
1672                 if (!itag) {
1673                         rth = rcu_dereference(FIB_RES_NH(res).nh_rth_input);
1674                         if (rt_cache_valid(rth)) {
1675                                 skb_dst_set_noref(skb, &rth->dst);
1676                                 err = 0;
1677                                 goto out;
1678                         }
1679                         do_cache = true;
1680                 }
1681         }
1682
1683         rth = rt_dst_alloc(net->loopback_dev,
1684                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
1685         if (!rth)
1686                 goto e_nobufs;
1687
1688         rth->dst.input= ip_local_deliver;
1689         rth->dst.output= ip_rt_bug;
1690 #ifdef CONFIG_IP_ROUTE_CLASSID
1691         rth->dst.tclassid = itag;
1692 #endif
1693
1694         rth->rt_genid = rt_genid(net);
1695         rth->rt_flags   = flags|RTCF_LOCAL;
1696         rth->rt_type    = res.type;
1697         rth->rt_is_input = 1;
1698         rth->rt_iif     = 0;
1699         rth->rt_pmtu    = 0;
1700         rth->rt_gateway = 0;
1701         rth->rt_uses_gateway = 0;
1702         INIT_LIST_HEAD(&rth->rt_uncached);
1703         RT_CACHE_STAT_INC(in_slow_tot);
1704         if (res.type == RTN_UNREACHABLE) {
1705                 rth->dst.input= ip_error;
1706                 rth->dst.error= -err;
1707                 rth->rt_flags   &= ~RTCF_LOCAL;
1708         }
1709         if (do_cache) {
1710                 if (unlikely(!rt_cache_route(&FIB_RES_NH(res), rth))) {
1711                         rth->dst.flags |= DST_NOCACHE;
1712                         rt_add_uncached_list(rth);
1713                 }
1714         }
1715         skb_dst_set(skb, &rth->dst);
1716         err = 0;
1717         goto out;
1718
1719 no_route:
1720         RT_CACHE_STAT_INC(in_no_route);
1721         res.type = RTN_UNREACHABLE;
1722         if (err == -ESRCH)
1723                 err = -ENETUNREACH;
1724         goto local_input;
1725
1726         /*
1727          *      Do not cache martian addresses: they should be logged (RFC1812)
1728          */
1729 martian_destination:
1730         RT_CACHE_STAT_INC(in_martian_dst);
1731 #ifdef CONFIG_IP_ROUTE_VERBOSE
1732         if (IN_DEV_LOG_MARTIANS(in_dev))
1733                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
1734                                      &daddr, &saddr, dev->name);
1735 #endif
1736
1737 e_inval:
1738         err = -EINVAL;
1739         goto out;
1740
1741 e_nobufs:
1742         err = -ENOBUFS;
1743         goto out;
1744
1745 martian_source:
1746         err = -EINVAL;
1747 martian_source_keep_err:
1748         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
1749         goto out;
1750 }
1751
1752 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1753                          u8 tos, struct net_device *dev)
1754 {
1755         int res;
1756
1757         rcu_read_lock();
1758
1759         /* Multicast recognition logic is moved from route cache to here.
1760            The problem was that too many Ethernet cards have broken/missing
1761            hardware multicast filters :-( As result the host on multicasting
1762            network acquires a lot of useless route cache entries, sort of
1763            SDR messages from all the world. Now we try to get rid of them.
1764            Really, provided software IP multicast filter is organized
1765            reasonably (at least, hashed), it does not result in a slowdown
1766            comparing with route cache reject entries.
1767            Note, that multicast routers are not affected, because
1768            route cache entry is created eventually.
1769          */
1770         if (ipv4_is_multicast(daddr)) {
1771                 struct in_device *in_dev = __in_dev_get_rcu(dev);
1772
1773                 if (in_dev) {
1774                         int our = ip_check_mc_rcu(in_dev, daddr, saddr,
1775                                                   ip_hdr(skb)->protocol);
1776                         if (our
1777 #ifdef CONFIG_IP_MROUTE
1778                                 ||
1779                             (!ipv4_is_local_multicast(daddr) &&
1780                              IN_DEV_MFORWARD(in_dev))
1781 #endif
1782                            ) {
1783                                 int res = ip_route_input_mc(skb, daddr, saddr,
1784                                                             tos, dev, our);
1785                                 rcu_read_unlock();
1786                                 return res;
1787                         }
1788                 }
1789                 rcu_read_unlock();
1790                 return -EINVAL;
1791         }
1792         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
1793         rcu_read_unlock();
1794         return res;
1795 }
1796 EXPORT_SYMBOL(ip_route_input_noref);
1797
1798 /* called with rcu_read_lock() */
1799 static struct rtable *__mkroute_output(const struct fib_result *res,
1800                                        const struct flowi4 *fl4, int orig_oif,
1801                                        struct net_device *dev_out,
1802                                        unsigned int flags)
1803 {
1804         struct fib_info *fi = res->fi;
1805         struct fib_nh_exception *fnhe;
1806         struct in_device *in_dev;
1807         u16 type = res->type;
1808         struct rtable *rth;
1809         bool do_cache;
1810
1811         in_dev = __in_dev_get_rcu(dev_out);
1812         if (!in_dev)
1813                 return ERR_PTR(-EINVAL);
1814
1815         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1816                 if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
1817                         return ERR_PTR(-EINVAL);
1818
1819         if (ipv4_is_lbcast(fl4->daddr))
1820                 type = RTN_BROADCAST;
1821         else if (ipv4_is_multicast(fl4->daddr))
1822                 type = RTN_MULTICAST;
1823         else if (ipv4_is_zeronet(fl4->daddr))
1824                 return ERR_PTR(-EINVAL);
1825
1826         if (dev_out->flags & IFF_LOOPBACK)
1827                 flags |= RTCF_LOCAL;
1828
1829         do_cache = true;
1830         if (type == RTN_BROADCAST) {
1831                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
1832                 fi = NULL;
1833         } else if (type == RTN_MULTICAST) {
1834                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
1835                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
1836                                      fl4->flowi4_proto))
1837                         flags &= ~RTCF_LOCAL;
1838                 else
1839                         do_cache = false;
1840                 /* If multicast route do not exist use
1841                  * default one, but do not gateway in this case.
1842                  * Yes, it is hack.
1843                  */
1844                 if (fi && res->prefixlen < 4)
1845                         fi = NULL;
1846         }
1847
1848         fnhe = NULL;
1849         do_cache &= fi != NULL;
1850         if (do_cache) {
1851                 struct rtable __rcu **prth;
1852                 struct fib_nh *nh = &FIB_RES_NH(*res);
1853
1854                 fnhe = find_exception(nh, fl4->daddr);
1855                 if (fnhe)
1856                         prth = &fnhe->fnhe_rth;
1857                 else {
1858                         if (unlikely(fl4->flowi4_flags &
1859                                      FLOWI_FLAG_KNOWN_NH &&
1860                                      !(nh->nh_gw &&
1861                                        nh->nh_scope == RT_SCOPE_LINK))) {
1862                                 do_cache = false;
1863                                 goto add;
1864                         }
1865                         prth = __this_cpu_ptr(nh->nh_pcpu_rth_output);
1866                 }
1867                 rth = rcu_dereference(*prth);
1868                 if (rt_cache_valid(rth)) {
1869                         dst_hold(&rth->dst);
1870                         return rth;
1871                 }
1872         }
1873
1874 add:
1875         rth = rt_dst_alloc(dev_out,
1876                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1877                            IN_DEV_CONF_GET(in_dev, NOXFRM),
1878                            do_cache);
1879         if (!rth)
1880                 return ERR_PTR(-ENOBUFS);
1881
1882         rth->dst.output = ip_output;
1883
1884         rth->rt_genid = rt_genid(dev_net(dev_out));
1885         rth->rt_flags   = flags;
1886         rth->rt_type    = type;
1887         rth->rt_is_input = 0;
1888         rth->rt_iif     = orig_oif ? : 0;
1889         rth->rt_pmtu    = 0;
1890         rth->rt_gateway = 0;
1891         rth->rt_uses_gateway = 0;
1892         INIT_LIST_HEAD(&rth->rt_uncached);
1893
1894         RT_CACHE_STAT_INC(out_slow_tot);
1895
1896         if (flags & RTCF_LOCAL)
1897                 rth->dst.input = ip_local_deliver;
1898         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
1899                 if (flags & RTCF_LOCAL &&
1900                     !(dev_out->flags & IFF_LOOPBACK)) {
1901                         rth->dst.output = ip_mc_output;
1902                         RT_CACHE_STAT_INC(out_slow_mc);
1903                 }
1904 #ifdef CONFIG_IP_MROUTE
1905                 if (type == RTN_MULTICAST) {
1906                         if (IN_DEV_MFORWARD(in_dev) &&
1907                             !ipv4_is_local_multicast(fl4->daddr)) {
1908                                 rth->dst.input = ip_mr_input;
1909                                 rth->dst.output = ip_mc_output;
1910                         }
1911                 }
1912 #endif
1913         }
1914
1915         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0);
1916
1917         return rth;
1918 }
1919
1920 /*
1921  * Major route resolver routine.
1922  */
1923
1924 struct rtable *__ip_route_output_key(struct net *net, struct flowi4 *fl4)
1925 {
1926         struct net_device *dev_out = NULL;
1927         __u8 tos = RT_FL_TOS(fl4);
1928         unsigned int flags = 0;
1929         struct fib_result res;
1930         struct rtable *rth;
1931         int orig_oif;
1932
1933         res.tclassid    = 0;
1934         res.fi          = NULL;
1935         res.table       = NULL;
1936
1937         orig_oif = fl4->flowi4_oif;
1938
1939         fl4->flowi4_iif = LOOPBACK_IFINDEX;
1940         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
1941         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
1942                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
1943
1944         rcu_read_lock();
1945         if (fl4->saddr) {
1946                 rth = ERR_PTR(-EINVAL);
1947                 if (ipv4_is_multicast(fl4->saddr) ||
1948                     ipv4_is_lbcast(fl4->saddr) ||
1949                     ipv4_is_zeronet(fl4->saddr))
1950                         goto out;
1951
1952                 /* I removed check for oif == dev_out->oif here.
1953                    It was wrong for two reasons:
1954                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
1955                       is assigned to multiple interfaces.
1956                    2. Moreover, we are allowed to send packets with saddr
1957                       of another iface. --ANK
1958                  */
1959
1960                 if (fl4->flowi4_oif == 0 &&
1961                     (ipv4_is_multicast(fl4->daddr) ||
1962                      ipv4_is_lbcast(fl4->daddr))) {
1963                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
1964                         dev_out = __ip_dev_find(net, fl4->saddr, false);
1965                         if (dev_out == NULL)
1966                                 goto out;
1967
1968                         /* Special hack: user can direct multicasts
1969                            and limited broadcast via necessary interface
1970                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
1971                            This hack is not just for fun, it allows
1972                            vic,vat and friends to work.
1973                            They bind socket to loopback, set ttl to zero
1974                            and expect that it will work.
1975                            From the viewpoint of routing cache they are broken,
1976                            because we are not allowed to build multicast path
1977                            with loopback source addr (look, routing cache
1978                            cannot know, that ttl is zero, so that packet
1979                            will not leave this host and route is valid).
1980                            Luckily, this hack is good workaround.
1981                          */
1982
1983                         fl4->flowi4_oif = dev_out->ifindex;
1984                         goto make_route;
1985                 }
1986
1987                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
1988                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
1989                         if (!__ip_dev_find(net, fl4->saddr, false))
1990                                 goto out;
1991                 }
1992         }
1993
1994
1995         if (fl4->flowi4_oif) {
1996                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
1997                 rth = ERR_PTR(-ENODEV);
1998                 if (dev_out == NULL)
1999                         goto out;
2000
2001                 /* RACE: Check return value of inet_select_addr instead. */
2002                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2003                         rth = ERR_PTR(-ENETUNREACH);
2004                         goto out;
2005                 }
2006                 if (ipv4_is_local_multicast(fl4->daddr) ||
2007                     ipv4_is_lbcast(fl4->daddr)) {
2008                         if (!fl4->saddr)
2009                                 fl4->saddr = inet_select_addr(dev_out, 0,
2010                                                               RT_SCOPE_LINK);
2011                         goto make_route;
2012                 }
2013                 if (!fl4->saddr) {
2014                         if (ipv4_is_multicast(fl4->daddr))
2015                                 fl4->saddr = inet_select_addr(dev_out, 0,
2016                                                               fl4->flowi4_scope);
2017                         else if (!fl4->daddr)
2018                                 fl4->saddr = inet_select_addr(dev_out, 0,
2019                                                               RT_SCOPE_HOST);
2020                 }
2021         }
2022
2023         if (!fl4->daddr) {
2024                 fl4->daddr = fl4->saddr;
2025                 if (!fl4->daddr)
2026                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2027                 dev_out = net->loopback_dev;
2028                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2029                 res.type = RTN_LOCAL;
2030                 flags |= RTCF_LOCAL;
2031                 goto make_route;
2032         }
2033
2034         if (fib_lookup(net, fl4, &res)) {
2035                 res.fi = NULL;
2036                 res.table = NULL;
2037                 if (fl4->flowi4_oif) {
2038                         /* Apparently, routing tables are wrong. Assume,
2039                            that the destination is on link.
2040
2041                            WHY? DW.
2042                            Because we are allowed to send to iface
2043                            even if it has NO routes and NO assigned
2044                            addresses. When oif is specified, routing
2045                            tables are looked up with only one purpose:
2046                            to catch if destination is gatewayed, rather than
2047                            direct. Moreover, if MSG_DONTROUTE is set,
2048                            we send packet, ignoring both routing tables
2049                            and ifaddr state. --ANK
2050
2051
2052                            We could make it even if oif is unknown,
2053                            likely IPv6, but we do not.
2054                          */
2055
2056                         if (fl4->saddr == 0)
2057                                 fl4->saddr = inet_select_addr(dev_out, 0,
2058                                                               RT_SCOPE_LINK);
2059                         res.type = RTN_UNICAST;
2060                         goto make_route;
2061                 }
2062                 rth = ERR_PTR(-ENETUNREACH);
2063                 goto out;
2064         }
2065
2066         if (res.type == RTN_LOCAL) {
2067                 if (!fl4->saddr) {
2068                         if (res.fi->fib_prefsrc)
2069                                 fl4->saddr = res.fi->fib_prefsrc;
2070                         else
2071                                 fl4->saddr = fl4->daddr;
2072                 }
2073                 dev_out = net->loopback_dev;
2074                 fl4->flowi4_oif = dev_out->ifindex;
2075                 flags |= RTCF_LOCAL;
2076                 goto make_route;
2077         }
2078
2079 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2080         if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0)
2081                 fib_select_multipath(&res);
2082         else
2083 #endif
2084         if (!res.prefixlen &&
2085             res.table->tb_num_default > 1 &&
2086             res.type == RTN_UNICAST && !fl4->flowi4_oif)
2087                 fib_select_default(&res);
2088
2089         if (!fl4->saddr)
2090                 fl4->saddr = FIB_RES_PREFSRC(net, res);
2091
2092         dev_out = FIB_RES_DEV(res);
2093         fl4->flowi4_oif = dev_out->ifindex;
2094
2095
2096 make_route:
2097         rth = __mkroute_output(&res, fl4, orig_oif, dev_out, flags);
2098
2099 out:
2100         rcu_read_unlock();
2101         return rth;
2102 }
2103 EXPORT_SYMBOL_GPL(__ip_route_output_key);
2104
2105 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2106 {
2107         return NULL;
2108 }
2109
2110 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2111 {
2112         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2113
2114         return mtu ? : dst->dev->mtu;
2115 }
2116
2117 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2118                                           struct sk_buff *skb, u32 mtu)
2119 {
2120 }
2121
2122 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2123                                        struct sk_buff *skb)
2124 {
2125 }
2126
2127 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2128                                           unsigned long old)
2129 {
2130         return NULL;
2131 }
2132
2133 static struct dst_ops ipv4_dst_blackhole_ops = {
2134         .family                 =       AF_INET,
2135         .protocol               =       cpu_to_be16(ETH_P_IP),
2136         .check                  =       ipv4_blackhole_dst_check,
2137         .mtu                    =       ipv4_blackhole_mtu,
2138         .default_advmss         =       ipv4_default_advmss,
2139         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2140         .redirect               =       ipv4_rt_blackhole_redirect,
2141         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2142         .neigh_lookup           =       ipv4_neigh_lookup,
2143 };
2144
2145 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2146 {
2147         struct rtable *ort = (struct rtable *) dst_orig;
2148         struct rtable *rt;
2149
2150         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_NONE, 0);
2151         if (rt) {
2152                 struct dst_entry *new = &rt->dst;
2153
2154                 new->__use = 1;
2155                 new->input = dst_discard;
2156                 new->output = dst_discard;
2157
2158                 new->dev = ort->dst.dev;
2159                 if (new->dev)
2160                         dev_hold(new->dev);
2161
2162                 rt->rt_is_input = ort->rt_is_input;
2163                 rt->rt_iif = ort->rt_iif;
2164                 rt->rt_pmtu = ort->rt_pmtu;
2165
2166                 rt->rt_genid = rt_genid(net);
2167                 rt->rt_flags = ort->rt_flags;
2168                 rt->rt_type = ort->rt_type;
2169                 rt->rt_gateway = ort->rt_gateway;
2170                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2171
2172                 INIT_LIST_HEAD(&rt->rt_uncached);
2173
2174                 dst_free(new);
2175         }
2176
2177         dst_release(dst_orig);
2178
2179         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2180 }
2181
2182 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2183                                     struct sock *sk)
2184 {
2185         struct rtable *rt = __ip_route_output_key(net, flp4);
2186
2187         if (IS_ERR(rt))
2188                 return rt;
2189
2190         if (flp4->flowi4_proto)
2191                 rt = (struct rtable *) xfrm_lookup(net, &rt->dst,
2192                                                    flowi4_to_flowi(flp4),
2193                                                    sk, 0);
2194
2195         return rt;
2196 }
2197 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2198
2199 static int rt_fill_info(struct net *net,  __be32 dst, __be32 src,
2200                         struct flowi4 *fl4, struct sk_buff *skb, u32 portid,
2201                         u32 seq, int event, int nowait, unsigned int flags)
2202 {
2203         struct rtable *rt = skb_rtable(skb);
2204         struct rtmsg *r;
2205         struct nlmsghdr *nlh;
2206         unsigned long expires = 0;
2207         u32 error;
2208         u32 metrics[RTAX_MAX];
2209
2210         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*r), flags);
2211         if (nlh == NULL)
2212                 return -EMSGSIZE;
2213
2214         r = nlmsg_data(nlh);
2215         r->rtm_family    = AF_INET;
2216         r->rtm_dst_len  = 32;
2217         r->rtm_src_len  = 0;
2218         r->rtm_tos      = fl4->flowi4_tos;
2219         r->rtm_table    = RT_TABLE_MAIN;
2220         if (nla_put_u32(skb, RTA_TABLE, RT_TABLE_MAIN))
2221                 goto nla_put_failure;
2222         r->rtm_type     = rt->rt_type;
2223         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2224         r->rtm_protocol = RTPROT_UNSPEC;
2225         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2226         if (rt->rt_flags & RTCF_NOTIFY)
2227                 r->rtm_flags |= RTM_F_NOTIFY;
2228
2229         if (nla_put_be32(skb, RTA_DST, dst))
2230                 goto nla_put_failure;
2231         if (src) {
2232                 r->rtm_src_len = 32;
2233                 if (nla_put_be32(skb, RTA_SRC, src))
2234                         goto nla_put_failure;
2235         }
2236         if (rt->dst.dev &&
2237             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2238                 goto nla_put_failure;
2239 #ifdef CONFIG_IP_ROUTE_CLASSID
2240         if (rt->dst.tclassid &&
2241             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2242                 goto nla_put_failure;
2243 #endif
2244         if (!rt_is_input_route(rt) &&
2245             fl4->saddr != src) {
2246                 if (nla_put_be32(skb, RTA_PREFSRC, fl4->saddr))
2247                         goto nla_put_failure;
2248         }
2249         if (rt->rt_uses_gateway &&
2250             nla_put_be32(skb, RTA_GATEWAY, rt->rt_gateway))
2251                 goto nla_put_failure;
2252
2253         expires = rt->dst.expires;
2254         if (expires) {
2255                 unsigned long now = jiffies;
2256
2257                 if (time_before(now, expires))
2258                         expires -= now;
2259                 else
2260                         expires = 0;
2261         }
2262
2263         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2264         if (rt->rt_pmtu && expires)
2265                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2266         if (rtnetlink_put_metrics(skb, metrics) < 0)
2267                 goto nla_put_failure;
2268
2269         if (fl4->flowi4_mark &&
2270             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2271                 goto nla_put_failure;
2272
2273         error = rt->dst.error;
2274
2275         if (rt_is_input_route(rt)) {
2276 #ifdef CONFIG_IP_MROUTE
2277                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2278                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2279                         int err = ipmr_get_route(net, skb,
2280                                                  fl4->saddr, fl4->daddr,
2281                                                  r, nowait);
2282                         if (err <= 0) {
2283                                 if (!nowait) {
2284                                         if (err == 0)
2285                                                 return 0;
2286                                         goto nla_put_failure;
2287                                 } else {
2288                                         if (err == -EMSGSIZE)
2289                                                 goto nla_put_failure;
2290                                         error = err;
2291                                 }
2292                         }
2293                 } else
2294 #endif
2295                         if (nla_put_u32(skb, RTA_IIF, skb->dev->ifindex))
2296                                 goto nla_put_failure;
2297         }
2298
2299         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2300                 goto nla_put_failure;
2301
2302         return nlmsg_end(skb, nlh);
2303
2304 nla_put_failure:
2305         nlmsg_cancel(skb, nlh);
2306         return -EMSGSIZE;
2307 }
2308
2309 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh)
2310 {
2311         struct net *net = sock_net(in_skb->sk);
2312         struct rtmsg *rtm;
2313         struct nlattr *tb[RTA_MAX+1];
2314         struct rtable *rt = NULL;
2315         struct flowi4 fl4;
2316         __be32 dst = 0;
2317         __be32 src = 0;
2318         u32 iif;
2319         int err;
2320         int mark;
2321         struct sk_buff *skb;
2322
2323         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2324         if (err < 0)
2325                 goto errout;
2326
2327         rtm = nlmsg_data(nlh);
2328
2329         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2330         if (skb == NULL) {
2331                 err = -ENOBUFS;
2332                 goto errout;
2333         }
2334
2335         /* Reserve room for dummy headers, this skb can pass
2336            through good chunk of routing engine.
2337          */
2338         skb_reset_mac_header(skb);
2339         skb_reset_network_header(skb);
2340
2341         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2342         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2343         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2344
2345         src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2346         dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2347         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2348         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2349
2350         memset(&fl4, 0, sizeof(fl4));
2351         fl4.daddr = dst;
2352         fl4.saddr = src;
2353         fl4.flowi4_tos = rtm->rtm_tos;
2354         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2355         fl4.flowi4_mark = mark;
2356
2357         if (iif) {
2358                 struct net_device *dev;
2359
2360                 dev = __dev_get_by_index(net, iif);
2361                 if (dev == NULL) {
2362                         err = -ENODEV;
2363                         goto errout_free;
2364                 }
2365
2366                 skb->protocol   = htons(ETH_P_IP);
2367                 skb->dev        = dev;
2368                 skb->mark       = mark;
2369                 local_bh_disable();
2370                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2371                 local_bh_enable();
2372
2373                 rt = skb_rtable(skb);
2374                 if (err == 0 && rt->dst.error)
2375                         err = -rt->dst.error;
2376         } else {
2377                 rt = ip_route_output_key(net, &fl4);
2378
2379                 err = 0;
2380                 if (IS_ERR(rt))
2381                         err = PTR_ERR(rt);
2382         }
2383
2384         if (err)
2385                 goto errout_free;
2386
2387         skb_dst_set(skb, &rt->dst);
2388         if (rtm->rtm_flags & RTM_F_NOTIFY)
2389                 rt->rt_flags |= RTCF_NOTIFY;
2390
2391         err = rt_fill_info(net, dst, src, &fl4, skb,
2392                            NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
2393                            RTM_NEWROUTE, 0, 0);
2394         if (err <= 0)
2395                 goto errout_free;
2396
2397         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2398 errout:
2399         return err;
2400
2401 errout_free:
2402         kfree_skb(skb);
2403         goto errout;
2404 }
2405
2406 int ip_rt_dump(struct sk_buff *skb,  struct netlink_callback *cb)
2407 {
2408         return skb->len;
2409 }
2410
2411 void ip_rt_multicast_event(struct in_device *in_dev)
2412 {
2413         rt_cache_flush(dev_net(in_dev->dev));
2414 }
2415
2416 #ifdef CONFIG_SYSCTL
2417 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
2418 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
2419 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
2420 static int ip_rt_gc_elasticity __read_mostly    = 8;
2421
2422 static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
2423                                         void __user *buffer,
2424                                         size_t *lenp, loff_t *ppos)
2425 {
2426         if (write) {
2427                 rt_cache_flush((struct net *)__ctl->extra1);
2428                 return 0;
2429         }
2430
2431         return -EINVAL;
2432 }
2433
2434 static ctl_table ipv4_route_table[] = {
2435         {
2436                 .procname       = "gc_thresh",
2437                 .data           = &ipv4_dst_ops.gc_thresh,
2438                 .maxlen         = sizeof(int),
2439                 .mode           = 0644,
2440                 .proc_handler   = proc_dointvec,
2441         },
2442         {
2443                 .procname       = "max_size",
2444                 .data           = &ip_rt_max_size,
2445                 .maxlen         = sizeof(int),
2446                 .mode           = 0644,
2447                 .proc_handler   = proc_dointvec,
2448         },
2449         {
2450                 /*  Deprecated. Use gc_min_interval_ms */
2451
2452                 .procname       = "gc_min_interval",
2453                 .data           = &ip_rt_gc_min_interval,
2454                 .maxlen         = sizeof(int),
2455                 .mode           = 0644,
2456                 .proc_handler   = proc_dointvec_jiffies,
2457         },
2458         {
2459                 .procname       = "gc_min_interval_ms",
2460                 .data           = &ip_rt_gc_min_interval,
2461                 .maxlen         = sizeof(int),
2462                 .mode           = 0644,
2463                 .proc_handler   = proc_dointvec_ms_jiffies,
2464         },
2465         {
2466                 .procname       = "gc_timeout",
2467                 .data           = &ip_rt_gc_timeout,
2468                 .maxlen         = sizeof(int),
2469                 .mode           = 0644,
2470                 .proc_handler   = proc_dointvec_jiffies,
2471         },
2472         {
2473                 .procname       = "gc_interval",
2474                 .data           = &ip_rt_gc_interval,
2475                 .maxlen         = sizeof(int),
2476                 .mode           = 0644,
2477                 .proc_handler   = proc_dointvec_jiffies,
2478         },
2479         {
2480                 .procname       = "redirect_load",
2481                 .data           = &ip_rt_redirect_load,
2482                 .maxlen         = sizeof(int),
2483                 .mode           = 0644,
2484                 .proc_handler   = proc_dointvec,
2485         },
2486         {
2487                 .procname       = "redirect_number",
2488                 .data           = &ip_rt_redirect_number,
2489                 .maxlen         = sizeof(int),
2490                 .mode           = 0644,
2491                 .proc_handler   = proc_dointvec,
2492         },
2493         {
2494                 .procname       = "redirect_silence",
2495                 .data           = &ip_rt_redirect_silence,
2496                 .maxlen         = sizeof(int),
2497                 .mode           = 0644,
2498                 .proc_handler   = proc_dointvec,
2499         },
2500         {
2501                 .procname       = "error_cost",
2502                 .data           = &ip_rt_error_cost,
2503                 .maxlen         = sizeof(int),
2504                 .mode           = 0644,
2505                 .proc_handler   = proc_dointvec,
2506         },
2507         {
2508                 .procname       = "error_burst",
2509                 .data           = &ip_rt_error_burst,
2510                 .maxlen         = sizeof(int),
2511                 .mode           = 0644,
2512                 .proc_handler   = proc_dointvec,
2513         },
2514         {
2515                 .procname       = "gc_elasticity",
2516                 .data           = &ip_rt_gc_elasticity,
2517                 .maxlen         = sizeof(int),
2518                 .mode           = 0644,
2519                 .proc_handler   = proc_dointvec,
2520         },
2521         {
2522                 .procname       = "mtu_expires",
2523                 .data           = &ip_rt_mtu_expires,
2524                 .maxlen         = sizeof(int),
2525                 .mode           = 0644,
2526                 .proc_handler   = proc_dointvec_jiffies,
2527         },
2528         {
2529                 .procname       = "min_pmtu",
2530                 .data           = &ip_rt_min_pmtu,
2531                 .maxlen         = sizeof(int),
2532                 .mode           = 0644,
2533                 .proc_handler   = proc_dointvec,
2534         },
2535         {
2536                 .procname       = "min_adv_mss",
2537                 .data           = &ip_rt_min_advmss,
2538                 .maxlen         = sizeof(int),
2539                 .mode           = 0644,
2540                 .proc_handler   = proc_dointvec,
2541         },
2542         { }
2543 };
2544
2545 static struct ctl_table ipv4_route_flush_table[] = {
2546         {
2547                 .procname       = "flush",
2548                 .maxlen         = sizeof(int),
2549                 .mode           = 0200,
2550                 .proc_handler   = ipv4_sysctl_rtcache_flush,
2551         },
2552         { },
2553 };
2554
2555 static __net_init int sysctl_route_net_init(struct net *net)
2556 {
2557         struct ctl_table *tbl;
2558
2559         tbl = ipv4_route_flush_table;
2560         if (!net_eq(net, &init_net)) {
2561                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
2562                 if (tbl == NULL)
2563                         goto err_dup;
2564
2565                 /* Don't export sysctls to unprivileged users */
2566                 if (net->user_ns != &init_user_ns)
2567                         tbl[0].procname = NULL;
2568         }
2569         tbl[0].extra1 = net;
2570
2571         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
2572         if (net->ipv4.route_hdr == NULL)
2573                 goto err_reg;
2574         return 0;
2575
2576 err_reg:
2577         if (tbl != ipv4_route_flush_table)
2578                 kfree(tbl);
2579 err_dup:
2580         return -ENOMEM;
2581 }
2582
2583 static __net_exit void sysctl_route_net_exit(struct net *net)
2584 {
2585         struct ctl_table *tbl;
2586
2587         tbl = net->ipv4.route_hdr->ctl_table_arg;
2588         unregister_net_sysctl_table(net->ipv4.route_hdr);
2589         BUG_ON(tbl == ipv4_route_flush_table);
2590         kfree(tbl);
2591 }
2592
2593 static __net_initdata struct pernet_operations sysctl_route_ops = {
2594         .init = sysctl_route_net_init,
2595         .exit = sysctl_route_net_exit,
2596 };
2597 #endif
2598
2599 static __net_init int rt_genid_init(struct net *net)
2600 {
2601         atomic_set(&net->rt_genid, 0);
2602         get_random_bytes(&net->ipv4.dev_addr_genid,
2603                          sizeof(net->ipv4.dev_addr_genid));
2604         return 0;
2605 }
2606
2607 static __net_initdata struct pernet_operations rt_genid_ops = {
2608         .init = rt_genid_init,
2609 };
2610
2611 static int __net_init ipv4_inetpeer_init(struct net *net)
2612 {
2613         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
2614
2615         if (!bp)
2616                 return -ENOMEM;
2617         inet_peer_base_init(bp);
2618         net->ipv4.peers = bp;
2619         return 0;
2620 }
2621
2622 static void __net_exit ipv4_inetpeer_exit(struct net *net)
2623 {
2624         struct inet_peer_base *bp = net->ipv4.peers;
2625
2626         net->ipv4.peers = NULL;
2627         inetpeer_invalidate_tree(bp);
2628         kfree(bp);
2629 }
2630
2631 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
2632         .init   =       ipv4_inetpeer_init,
2633         .exit   =       ipv4_inetpeer_exit,
2634 };
2635
2636 #ifdef CONFIG_IP_ROUTE_CLASSID
2637 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
2638 #endif /* CONFIG_IP_ROUTE_CLASSID */
2639
2640 int __init ip_rt_init(void)
2641 {
2642         int rc = 0;
2643
2644         ip_idents = kmalloc(IP_IDENTS_SZ * sizeof(*ip_idents), GFP_KERNEL);
2645         if (!ip_idents)
2646                 panic("IP: failed to allocate ip_idents\n");
2647
2648         prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
2649
2650 #ifdef CONFIG_IP_ROUTE_CLASSID
2651         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
2652         if (!ip_rt_acct)
2653                 panic("IP: failed to allocate ip_rt_acct\n");
2654 #endif
2655
2656         ipv4_dst_ops.kmem_cachep =
2657                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
2658                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2659
2660         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
2661
2662         if (dst_entries_init(&ipv4_dst_ops) < 0)
2663                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
2664
2665         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
2666                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
2667
2668         ipv4_dst_ops.gc_thresh = ~0;
2669         ip_rt_max_size = INT_MAX;
2670
2671         devinet_init();
2672         ip_fib_init();
2673
2674         if (ip_rt_proc_init())
2675                 pr_err("Unable to create route proc files\n");
2676 #ifdef CONFIG_XFRM
2677         xfrm_init();
2678         xfrm4_init();
2679 #endif
2680         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
2681
2682 #ifdef CONFIG_SYSCTL
2683         register_pernet_subsys(&sysctl_route_ops);
2684 #endif
2685         register_pernet_subsys(&rt_genid_ops);
2686         register_pernet_subsys(&ipv4_inetpeer_ops);
2687         return rc;
2688 }
2689
2690 #ifdef CONFIG_SYSCTL
2691 /*
2692  * We really need to sanitize the damn ipv4 init order, then all
2693  * this nonsense will go away.
2694  */
2695 void __init ip_static_sysctl_init(void)
2696 {
2697         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
2698 }
2699 #endif