58 #define for_nexthops(fi) {						\  argument
60 	for (nhsel = 0, nh = (fi)->fib_nh;				\
61 	     nhsel < fib_info_num_path((fi));				\
64 #define change_nexthops(fi) {						\  argument
66 	for (nhsel = 0,	nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
67 	     nhsel < fib_info_num_path((fi));				\
74 #define for_nexthops(fi) {						\  argument
75 	int nhsel; const struct fib_nh *nh = (fi)->fib_nh;		\
78 #define change_nexthops(fi) {						\  argument
80 	struct fib_nh *nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
85 #define endfor_nexthops(fi) }  argument
224 	struct fib_info *fi = container_of(head, struct fib_info, rcu);  in free_fib_info_rcu()  local
226 	if (fi->nh) {  in free_fib_info_rcu()
227 		nexthop_put(fi->nh);  in free_fib_info_rcu()
229 		change_nexthops(fi) {  in free_fib_info_rcu()
230 			fib_nh_release(fi->fib_net, nexthop_nh);  in free_fib_info_rcu()
231 		} endfor_nexthops(fi);  in free_fib_info_rcu()
234 	ip_fib_metrics_put(fi->fib_metrics);  in free_fib_info_rcu()
236 	kfree(fi);  in free_fib_info_rcu()
239 void free_fib_info(struct fib_info *fi)  in free_fib_info()  argument
241 	if (fi->fib_dead == 0) {  in free_fib_info()
242 		pr_warn("Freeing alive fib_info %p\n", fi);  in free_fib_info()
246 	call_rcu_hurry(&fi->rcu, free_fib_info_rcu);  in free_fib_info()
250 void fib_release_info(struct fib_info *fi)  in fib_release_info()  argument
253 	if (fi && refcount_dec_and_test(&fi->fib_treeref)) {  in fib_release_info()
254 		hlist_del(&fi->fib_hash);  in fib_release_info()
255 		fi->fib_net->ipv4.fib_info_cnt--;  in fib_release_info()
257 		if (fi->fib_prefsrc)  in fib_release_info()
258 			hlist_del(&fi->fib_lhash);  in fib_release_info()
259 		if (fi->nh) {  in fib_release_info()
260 			list_del(&fi->nh_list);  in fib_release_info()
262 			change_nexthops(fi) {  in fib_release_info()
266 			} endfor_nexthops(fi)  in fib_release_info()
269 		WRITE_ONCE(fi->fib_dead, 1);  in fib_release_info()
270 		fib_info_put(fi);  in fib_release_info()
274 static inline int nh_comp(struct fib_info *fi, struct fib_info *ofi)  in nh_comp()  argument
278 	if (fi->nh || ofi->nh)  in nh_comp()
279 		return nexthop_cmp(fi->nh, ofi->nh) ? 0 : -1;  in nh_comp()
284 	for_nexthops(fi) {  in nh_comp()
307 	} endfor_nexthops(fi);  in nh_comp()
334 static struct hlist_head *fib_info_hash_bucket(struct fib_info *fi)  in fib_info_hash_bucket()  argument
336 	struct net *net = fi->fib_net;  in fib_info_hash_bucket()
339 	val = fib_info_hashfn_1(fi->fib_nhs, fi->fib_protocol,  in fib_info_hash_bucket()
340 				fi->fib_scope, (__force u32)fi->fib_prefsrc,  in fib_info_hash_bucket()
341 				fi->fib_priority);  in fib_info_hash_bucket()
343 	if (fi->nh) {  in fib_info_hash_bucket()
344 		val ^= fi->nh->id;  in fib_info_hash_bucket()
346 		for_nexthops(fi) {  in fib_info_hash_bucket()
348 		} endfor_nexthops(fi)  in fib_info_hash_bucket()
397 		struct fib_info *fi;  in fib_info_hash_grow()  local
399 		hlist_for_each_entry_safe(fi, n, head, fib_hash)  in fib_info_hash_grow()
400 			hlist_add_head(&fi->fib_hash, fib_info_hash_bucket(fi));  in fib_info_hash_grow()
406 		struct fib_info *fi;  in fib_info_hash_grow()  local
408 		hlist_for_each_entry_safe(fi, n, lhead, fib_lhash)  in fib_info_hash_grow()
409 			hlist_add_head(&fi->fib_lhash,  in fib_info_hash_grow()
410 				       fib_info_laddrhash_bucket(fi->fib_net,  in fib_info_hash_grow()
411 								 fi->fib_prefsrc));  in fib_info_hash_grow()
422 	struct fib_info *fi;  in fib_find_info_nh()  local
432 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info_nh()
433 		if (!fi->nh || fi->nh->id != cfg->fc_nh_id)  in fib_find_info_nh()
436 		if (cfg->fc_protocol == fi->fib_protocol &&  in fib_find_info_nh()
437 		    cfg->fc_scope == fi->fib_scope &&  in fib_find_info_nh()
438 		    cfg->fc_prefsrc == fi->fib_prefsrc &&  in fib_find_info_nh()
439 		    cfg->fc_priority == fi->fib_priority &&  in fib_find_info_nh()
440 		    cfg->fc_type == fi->fib_type &&  in fib_find_info_nh()
441 		    cfg->fc_table == fi->fib_tb_id &&  in fib_find_info_nh()
442 		    !((cfg->fc_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK))  in fib_find_info_nh()
443 			return fi;  in fib_find_info_nh()
452 	struct fib_info *fi;  in fib_find_info()  local
454 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info()
455 		if (fi->fib_nhs != nfi->fib_nhs)  in fib_find_info()
458 		if (nfi->fib_protocol == fi->fib_protocol &&  in fib_find_info()
459 		    nfi->fib_scope == fi->fib_scope &&  in fib_find_info()
460 		    nfi->fib_prefsrc == fi->fib_prefsrc &&  in fib_find_info()
461 		    nfi->fib_priority == fi->fib_priority &&  in fib_find_info()
462 		    nfi->fib_type == fi->fib_type &&  in fib_find_info()
463 		    nfi->fib_tb_id == fi->fib_tb_id &&  in fib_find_info()
464 		    memcmp(nfi->fib_metrics, fi->fib_metrics,  in fib_find_info()
466 		    !((nfi->fib_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK) &&  in fib_find_info()
467 		    nh_comp(fi, nfi) == 0)  in fib_find_info()
468 			return fi;  in fib_find_info()
495 size_t fib_nlmsg_size(struct fib_info *fi)  in fib_nlmsg_size()  argument
503 	unsigned int nhs = fib_info_num_path(fi);  in fib_nlmsg_size()
508 	if (fi->nh)  in fib_nlmsg_size()
523 		for (i = 0; i < fib_info_num_path(fi); i++) {  in fib_nlmsg_size()
524 			struct fib_nh_common *nhc = fib_info_nhc(fi, i);  in fib_nlmsg_size()
556 	fri.fi = fa->fa_info;  in rtmsg_fib()
579 static int fib_detect_death(struct fib_info *fi, int order,  in fib_detect_death()  argument
583 	const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);  in fib_detect_death()
607 		*last_resort = fi;  in fib_detect_death()
720 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
724 	struct net *net = fi->fib_net;  in fib_get_nhs()
729 	change_nexthops(fi) {  in fib_get_nhs()
798 	} endfor_nexthops(fi);  in fib_get_nhs()
801 	nh = fib_info_nh(fi, 0);  in fib_get_nhs()
831 static void fib_rebalance(struct fib_info *fi)  in fib_rebalance()  argument
836 	if (fib_info_num_path(fi) < 2)  in fib_rebalance()
840 	for_nexthops(fi) {  in fib_rebalance()
849 	} endfor_nexthops(fi);  in fib_rebalance()
852 	change_nexthops(fi) {  in fib_rebalance()
867 	} endfor_nexthops(fi);  in fib_rebalance()
871 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
880 #define fib_rebalance(fi) do { } while (0)  argument
906 int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi,  in fib_nh_match()  argument
914 	if (cfg->fc_priority && cfg->fc_priority != fi->fib_priority)  in fib_nh_match()
918 		if (fi->nh && cfg->fc_nh_id == fi->nh->id)  in fib_nh_match()
923 	if (fi->nh) {  in fib_nh_match()
932 		nh = fib_info_nh(fi, 0);  in fib_nh_match()
966 	for_nexthops(fi) {  in fib_nh_match()
1034 	} endfor_nexthops(fi);  in fib_nh_match()
1039 bool fib_metrics_match(struct fib_config *cfg, struct fib_info *fi)  in fib_metrics_match()  argument
1069 		fi_val = fi->fib_metrics->metrics[type - 1];  in fib_metrics_match()
1315 	if (res->fi->fib_prefsrc)  in fib_result_prefsrc()
1316 		return res->fi->fib_prefsrc;  in fib_result_prefsrc()
1327 	return fib_info_update_nhc_saddr(net, nhc, res->fi->fib_scope);  in fib_result_prefsrc()
1358 	struct fib_info *fi = NULL;  in fib_create_info()  local
1382 			fi = fib_find_info_nh(net, cfg);  in fib_create_info()
1383 			if (fi) {  in fib_create_info()
1384 				refcount_inc(&fi->fib_treeref);  in fib_create_info()
1385 				return fi;  in fib_create_info()
1407 	fi = kzalloc(struct_size(fi, fib_nh, nhs), GFP_KERNEL);  in fib_create_info()
1408 	if (!fi) {  in fib_create_info()
1413 	fi->fib_metrics = ip_fib_metrics_init(cfg->fc_mx, cfg->fc_mx_len, extack);  in fib_create_info()
1414 	if (IS_ERR(fi->fib_metrics)) {  in fib_create_info()
1415 		err = PTR_ERR(fi->fib_metrics);  in fib_create_info()
1416 		kfree(fi);  in fib_create_info()
1420 	fi->fib_net = net;  in fib_create_info()
1421 	fi->fib_protocol = cfg->fc_protocol;  in fib_create_info()
1422 	fi->fib_scope = cfg->fc_scope;  in fib_create_info()
1423 	fi->fib_flags = cfg->fc_flags;  in fib_create_info()
1424 	fi->fib_priority = cfg->fc_priority;  in fib_create_info()
1425 	fi->fib_prefsrc = cfg->fc_prefsrc;  in fib_create_info()
1426 	fi->fib_type = cfg->fc_type;  in fib_create_info()
1427 	fi->fib_tb_id = cfg->fc_table;  in fib_create_info()
1429 	fi->fib_nhs = nhs;  in fib_create_info()
1436 			fi->nh = nh;  in fib_create_info()
1439 		change_nexthops(fi) {  in fib_create_info()
1440 			nexthop_nh->nh_parent = fi;  in fib_create_info()
1441 		} endfor_nexthops(fi)  in fib_create_info()
1444 			err = fib_get_nhs(fi, cfg->fc_mp, cfg->fc_mp_len, cfg,  in fib_create_info()
1447 			err = fib_nh_init(net, fi->fib_nh, cfg, 1, extack);  in fib_create_info()
1479 	if (fi->nh) {  in fib_create_info()
1480 		err = fib_check_nexthop(fi->nh, cfg->fc_scope, extack);  in fib_create_info()
1484 		struct fib_nh *nh = fi->fib_nh;  in fib_create_info()
1507 		change_nexthops(fi) {  in fib_create_info()
1515 		} endfor_nexthops(fi)  in fib_create_info()
1516 		if (linkdown == fi->fib_nhs)  in fib_create_info()
1517 			fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_create_info()
1520 	if (fi->fib_prefsrc && !fib_valid_prefsrc(cfg, fi->fib_prefsrc)) {  in fib_create_info()
1525 	if (!fi->nh) {  in fib_create_info()
1526 		change_nexthops(fi) {  in fib_create_info()
1528 						  fi->fib_scope);  in fib_create_info()
1530 				fi->fib_nh_is_v6 = true;  in fib_create_info()
1531 		} endfor_nexthops(fi)  in fib_create_info()
1533 		fib_rebalance(fi);  in fib_create_info()
1537 	ofi = fib_find_info(fi);  in fib_create_info()
1539 		/* fib_table_lookup() should not see @fi yet. */  in fib_create_info()
1540 		fi->fib_dead = 1;  in fib_create_info()
1541 		free_fib_info(fi);  in fib_create_info()
1546 	refcount_set(&fi->fib_treeref, 1);  in fib_create_info()
1547 	refcount_set(&fi->fib_clntref, 1);  in fib_create_info()
1550 	hlist_add_head(&fi->fib_hash, fib_info_hash_bucket(fi));  in fib_create_info()
1552 	if (fi->fib_prefsrc) {  in fib_create_info()
1555 		head = fib_info_laddrhash_bucket(net, fi->fib_prefsrc);  in fib_create_info()
1556 		hlist_add_head(&fi->fib_lhash, head);  in fib_create_info()
1558 	if (fi->nh) {  in fib_create_info()
1559 		list_add(&fi->nh_list, &nh->fi_list);  in fib_create_info()
1561 		change_nexthops(fi) {  in fib_create_info()
1568 		} endfor_nexthops(fi)  in fib_create_info()
1570 	return fi;  in fib_create_info()
1576 	if (fi) {  in fib_create_info()
1577 		/* fib_table_lookup() should not see @fi yet. */  in fib_create_info()
1578 		fi->fib_dead = 1;  in fib_create_info()
1579 		free_fib_info(fi);  in fib_create_info()
1690 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)  in fib_add_multipath()  argument
1698 	if (unlikely(fi->nh)) {  in fib_add_multipath()
1699 		if (nexthop_mpath_fill_node(skb, fi->nh, AF_INET) < 0)  in fib_add_multipath()
1704 	for_nexthops(fi) {  in fib_add_multipath()
1712 	} endfor_nexthops(fi);  in fib_add_multipath()
1723 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)  in fib_add_multipath()  argument
1732 	unsigned int nhs = fib_info_num_path(fri->fi);  in fib_dump_info()
1733 	struct fib_info *fi = fri->fi;  in fib_dump_info()  local
1754 	rtm->rtm_flags = fi->fib_flags;  in fib_dump_info()
1755 	rtm->rtm_scope = fi->fib_scope;  in fib_dump_info()
1756 	rtm->rtm_protocol = fi->fib_protocol;  in fib_dump_info()
1761 	if (fi->fib_priority &&  in fib_dump_info()
1762 	    nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority))  in fib_dump_info()
1764 	if (rtnetlink_put_metrics(skb, fi->fib_metrics->metrics) < 0)  in fib_dump_info()
1767 	if (fi->fib_prefsrc &&  in fib_dump_info()
1768 	    nla_put_in_addr(skb, RTA_PREFSRC, fi->fib_prefsrc))  in fib_dump_info()
1771 	if (fi->nh) {  in fib_dump_info()
1772 		if (nla_put_u32(skb, RTA_NH_ID, fi->nh->id))  in fib_dump_info()
1774 		if (nexthop_is_blackhole(fi->nh))  in fib_dump_info()
1776 		if (!READ_ONCE(fi->fib_net->ipv4.sysctl_nexthop_compat_mode))  in fib_dump_info()
1781 		const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);  in fib_dump_info()
1799 		if (fib_add_multipath(skb, fi) < 0)  in fib_dump_info()
1830 	struct fib_info *fi;  in fib_sync_down_addr()  local
1837 	hlist_for_each_entry(fi, head, fib_lhash) {  in fib_sync_down_addr()
1838 		if (!net_eq(fi->fib_net, net) ||  in fib_sync_down_addr()
1839 		    fi->fib_tb_id != tb_id)  in fib_sync_down_addr()
1841 		if (fi->fib_prefsrc == local) {  in fib_sync_down_addr()
1842 			fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_addr()
1843 			fi->pfsrc_removed = true;  in fib_sync_down_addr()
1948 		struct fib_info *fi = nh->nh_parent;  in fib_sync_down_dev()  local
1951 		BUG_ON(!fi->fib_nhs);  in fib_sync_down_dev()
1953 		if (fi == prev_fi)  in fib_sync_down_dev()
1955 		prev_fi = fi;  in fib_sync_down_dev()
1957 		change_nexthops(fi) {  in fib_sync_down_dev()
1978 				dead = fi->fib_nhs;  in fib_sync_down_dev()
1982 		} endfor_nexthops(fi)  in fib_sync_down_dev()
1983 		if (dead == fi->fib_nhs) {  in fib_sync_down_dev()
1987 				fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_dev()
1990 				fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_sync_down_dev()
1996 		fib_rebalance(fi);  in fib_sync_down_dev()
2005 	struct fib_info *fi = NULL, *last_resort = NULL;  in fib_select_default()  local
2011 	u32 last_prio = res->fi->fib_priority;  in fib_select_default()
2045 		if (!fi) {  in fib_select_default()
2046 			if (next_fi != res->fi)  in fib_select_default()
2049 		} else if (!fib_detect_death(fi, order, &last_resort,  in fib_select_default()
2051 			fib_result_assign(res, fi);  in fib_select_default()
2055 		fi = next_fi;  in fib_select_default()
2059 	if (order <= 0 || !fi) {  in fib_select_default()
2065 	if (!fib_detect_death(fi, order, &last_resort, &last_idx,  in fib_select_default()
2067 		fib_result_assign(res, fi);  in fib_select_default()
2107 		struct fib_info *fi = nh->nh_parent;  in fib_sync_up()  local
2110 		BUG_ON(!fi->fib_nhs);  in fib_sync_up()
2112 		if (fi == prev_fi)  in fib_sync_up()
2115 		prev_fi = fi;  in fib_sync_up()
2117 		change_nexthops(fi) {  in fib_sync_up()
2131 		} endfor_nexthops(fi)  in fib_sync_up()
2134 			fi->fib_flags &= ~nh_flags;  in fib_sync_up()
2138 		fib_rebalance(fi);  in fib_sync_up()
2173 	struct fib_info *fi = res->fi;  in fib_select_multipath()  local
2174 	struct net *net = fi->fib_net;  in fib_select_multipath()
2177 	if (unlikely(res->fi->nh)) {  in fib_select_multipath()
2182 	change_nexthops(fi) {  in fib_select_multipath()
2199 	} endfor_nexthops(fi);  in fib_select_multipath()
2210 	if (fib_info_num_path(res->fi) > 1) {  in fib_select_path()