[deliverable/linux.git] / net / ipv6 / ila / ila_xlat.c

#include <linux/jhash.h>
#include <linux/netfilter.h>
#include <linux/rcupdate.h>
#include <linux/rhashtable.h>
#include <linux/vmalloc.h>
#include <net/genetlink.h>
#include <net/ila.h>
#include <net/netns/generic.h>
#include <uapi/linux/genetlink.h>
#include "ila.h"

struct ila_xlat_params {
	struct ila_params ip;
	int ifindex;
};

struct ila_map {
	struct ila_xlat_params xp;
	struct rhash_head node;
	struct ila_map __rcu *next;
	struct rcu_head rcu;
};

static unsigned int ila_net_id;

struct ila_net {
	struct rhashtable rhash_table;
	spinlock_t *locks; /* Bucket locks for entry manipulation */
	unsigned int locks_mask;
	bool hooks_registered;
};

#define	LOCKS_PER_CPU 10

static int alloc_ila_locks(struct ila_net *ilan)
{
	unsigned int i, size;
	unsigned int nr_pcpus = num_possible_cpus();

	nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
	size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU);

	if (sizeof(spinlock_t) != 0) {
#ifdef CONFIG_NUMA
		if (size * sizeof(spinlock_t) > PAGE_SIZE)
			ilan->locks = vmalloc(size * sizeof(spinlock_t));
		else
#endif
		ilan->locks = kmalloc_array(size, sizeof(spinlock_t),
					    GFP_KERNEL);
		if (!ilan->locks)
			return -ENOMEM;
		for (i = 0; i < size; i++)
			spin_lock_init(&ilan->locks[i]);
	}
	ilan->locks_mask = size - 1;

	return 0;
}

static u32 hashrnd __read_mostly;
static __always_inline void __ila_hash_secret_init(void)
{
	net_get_random_once(&hashrnd, sizeof(hashrnd));
}

static inline u32 ila_locator_hash(struct ila_locator loc)
{
	u32 *v = (u32 *)loc.v32;

	return jhash_2words(v[0], v[1], hashrnd);
}

static inline spinlock_t *ila_get_lock(struct ila_net *ilan,
				       struct ila_locator loc)
{
	return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask];
}

static inline int ila_cmp_wildcards(struct ila_map *ila,
				    struct ila_addr *iaddr, int ifindex)
{
	return (ila->xp.ifindex && ila->xp.ifindex != ifindex);
}

static inline int ila_cmp_params(struct ila_map *ila,
				 struct ila_xlat_params *xp)
{
	return (ila->xp.ifindex != xp->ifindex);
}

static int ila_cmpfn(struct rhashtable_compare_arg *arg,
		     const void *obj)
{
	const struct ila_map *ila = obj;

	return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key);
}

static inline int ila_order(struct ila_map *ila)
{
	int score = 0;

	if (ila->xp.ifindex)
		score += 1 << 1;

	return score;
}

static const struct rhashtable_params rht_params = {
	.nelem_hint = 1024,
	.head_offset = offsetof(struct ila_map, node),
	.key_offset = offsetof(struct ila_map, xp.ip.locator_match),
	.key_len = sizeof(u64), /* identifier */
	.max_size = 1048576,
	.min_size = 256,
	.automatic_shrinking = true,
	.obj_cmpfn = ila_cmpfn,
};

static struct genl_family ila_nl_family = {
	.id		= GENL_ID_GENERATE,
	.hdrsize	= 0,
	.name		= ILA_GENL_NAME,
	.version	= ILA_GENL_VERSION,
	.maxattr	= ILA_ATTR_MAX,
	.netnsok	= true,
	.parallel_ops	= true,
};

static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
	[ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
	[ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
	[ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
	[ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, },
};

static int parse_nl_config(struct genl_info *info,
			   struct ila_xlat_params *xp)
{
	memset(xp, 0, sizeof(*xp));

	if (info->attrs[ILA_ATTR_LOCATOR])
		xp->ip.locator.v64 = (__force __be64)nla_get_u64(
			info->attrs[ILA_ATTR_LOCATOR]);

	if (info->attrs[ILA_ATTR_LOCATOR_MATCH])
		xp->ip.locator_match.v64 = (__force __be64)nla_get_u64(
			info->attrs[ILA_ATTR_LOCATOR_MATCH]);

	if (info->attrs[ILA_ATTR_CSUM_MODE])
		xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]);

	if (info->attrs[ILA_ATTR_IFINDEX])
		xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]);

	return 0;
}

/* Must be called with rcu readlock */
static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr,
						   int ifindex,
						   struct ila_net *ilan)
{
	struct ila_map *ila;

	ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc,
				     rht_params);
	while (ila) {
		if (!ila_cmp_wildcards(ila, iaddr, ifindex))
			return ila;
		ila = rcu_access_pointer(ila->next);
	}

	return NULL;
}

/* Must be called with rcu readlock */
static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp,
						   struct ila_net *ilan)
{
	struct ila_map *ila;

	ila = rhashtable_lookup_fast(&ilan->rhash_table,
				     &xp->ip.locator_match,
				     rht_params);
	while (ila) {
		if (!ila_cmp_params(ila, xp))
			return ila;
		ila = rcu_access_pointer(ila->next);
	}

	return NULL;
}

static inline void ila_release(struct ila_map *ila)
{
	kfree_rcu(ila, rcu);
}

static void ila_free_cb(void *ptr, void *arg)
{
	struct ila_map *ila = (struct ila_map *)ptr, *next;

	/* Assume rcu_readlock held */
	while (ila) {
		next = rcu_access_pointer(ila->next);
		ila_release(ila);
		ila = next;
	}
}

static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral);

static unsigned int
ila_nf_input(void *priv,
	     struct sk_buff *skb,
	     const struct nf_hook_state *state)
{
	ila_xlat_addr(skb, false);
	return NF_ACCEPT;
}

static struct nf_hook_ops ila_nf_hook_ops[] __read_mostly = {
	{
		.hook = ila_nf_input,
		.pf = NFPROTO_IPV6,
		.hooknum = NF_INET_PRE_ROUTING,
		.priority = -1,
	},
};

static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
{
	struct ila_net *ilan = net_generic(net, ila_net_id);
	struct ila_map *ila, *head;
	spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
	int err = 0, order;

	if (!ilan->hooks_registered) {
		/* We defer registering net hooks in the namespace until the
		 * first mapping is added.
		 */
		err = nf_register_net_hooks(net, ila_nf_hook_ops,
					    ARRAY_SIZE(ila_nf_hook_ops));
		if (err)
			return err;

		ilan->hooks_registered = true;
	}

	ila = kzalloc(sizeof(*ila), GFP_KERNEL);
	if (!ila)
		return -ENOMEM;

	ila_init_saved_csum(&xp->ip);

	ila->xp = *xp;

	order = ila_order(ila);

	spin_lock(lock);

	head = rhashtable_lookup_fast(&ilan->rhash_table,
				      &xp->ip.locator_match,
				      rht_params);
	if (!head) {
		/* New entry for the rhash_table */
		err = rhashtable_lookup_insert_fast(&ilan->rhash_table,
						    &ila->node, rht_params);
	} else {
		struct ila_map *tila = head, *prev = NULL;

		do {
			if (!ila_cmp_params(tila, xp)) {
				err = -EEXIST;
				goto out;
			}

			if (order > ila_order(tila))
				break;

			prev = tila;
			tila = rcu_dereference_protected(tila->next,
				lockdep_is_held(lock));
		} while (tila);

		if (prev) {
			/* Insert in sub list of head */
			RCU_INIT_POINTER(ila->next, tila);
			rcu_assign_pointer(prev->next, ila);
		} else {
			/* Make this ila new head */
			RCU_INIT_POINTER(ila->next, head);
			err = rhashtable_replace_fast(&ilan->rhash_table,
						      &head->node,
						      &ila->node, rht_params);
			if (err)
				goto out;
		}
	}

out:
	spin_unlock(lock);

	if (err)
		kfree(ila);

	return err;
}

static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp)
{
	struct ila_net *ilan = net_generic(net, ila_net_id);
	struct ila_map *ila, *head, *prev;
	spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
	int err = -ENOENT;

	spin_lock(lock);

	head = rhashtable_lookup_fast(&ilan->rhash_table,
				      &xp->ip.locator_match, rht_params);
	ila = head;

	prev = NULL;

	while (ila) {
		if (ila_cmp_params(ila, xp)) {
			prev = ila;
			ila = rcu_dereference_protected(ila->next,
							lockdep_is_held(lock));
			continue;
		}

		err = 0;

		if (prev) {
			/* Not head, just delete from list */
			rcu_assign_pointer(prev->next, ila->next);
		} else {
			/* It is the head. If there is something in the
			 * sublist we need to make a new head.
			 */
			head = rcu_dereference_protected(ila->next,
							 lockdep_is_held(lock));
			if (head) {
				/* Put first entry in the sublist into the
				 * table
				 */
				err = rhashtable_replace_fast(
					&ilan->rhash_table, &ila->node,
					&head->node, rht_params);
				if (err)
					goto out;
			} else {
				/* Entry no longer used */
				err = rhashtable_remove_fast(&ilan->rhash_table,
							     &ila->node,
							     rht_params);
			}
		}

		ila_release(ila);

		break;
	}

out:
	spin_unlock(lock);

	return err;
}

static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
{
	struct net *net = genl_info_net(info);
	struct ila_xlat_params p;
	int err;

	err = parse_nl_config(info, &p);
	if (err)
		return err;

	return ila_add_mapping(net, &p);
}

static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
{
	struct net *net = genl_info_net(info);
	struct ila_xlat_params xp;
	int err;

	err = parse_nl_config(info, &xp);
	if (err)
		return err;

	ila_del_mapping(net, &xp);

	return 0;
}

static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg)
{
	if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR,
			      (__force u64)ila->xp.ip.locator.v64,
			      ILA_ATTR_PAD) ||
	    nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH,
			      (__force u64)ila->xp.ip.locator_match.v64,
			      ILA_ATTR_PAD) ||
	    nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) ||
	    nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode))
		return -1;

	return 0;
}

static int ila_dump_info(struct ila_map *ila,
			 u32 portid, u32 seq, u32 flags,
			 struct sk_buff *skb, u8 cmd)
{
	void *hdr;

	hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd);
	if (!hdr)
		return -ENOMEM;

	if (ila_fill_info(ila, skb) < 0)
		goto nla_put_failure;

	genlmsg_end(skb, hdr);
	return 0;

nla_put_failure:
	genlmsg_cancel(skb, hdr);
	return -EMSGSIZE;
}

static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
{
	struct net *net = genl_info_net(info);
	struct ila_net *ilan = net_generic(net, ila_net_id);
	struct sk_buff *msg;
	struct ila_xlat_params xp;
	struct ila_map *ila;
	int ret;

	ret = parse_nl_config(info, &xp);
	if (ret)
		return ret;

	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
	if (!msg)
		return -ENOMEM;

	rcu_read_lock();

	ila = ila_lookup_by_params(&xp, ilan);
	if (ila) {
		ret = ila_dump_info(ila,
				    info->snd_portid,
				    info->snd_seq, 0, msg,
				    info->genlhdr->cmd);
	}

	rcu_read_unlock();

	if (ret < 0)
		goto out_free;

	return genlmsg_reply(msg, info);

out_free:
	nlmsg_free(msg);
	return ret;
}

struct ila_dump_iter {
	struct rhashtable_iter rhiter;
};

static int ila_nl_dump_start(struct netlink_callback *cb)
{
	struct net *net = sock_net(cb->skb->sk);
	struct ila_net *ilan = net_generic(net, ila_net_id);
	struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args;

	return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter,
				    GFP_KERNEL);
}

static int ila_nl_dump_done(struct netlink_callback *cb)
{
	struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args;

	rhashtable_walk_exit(&iter->rhiter);

	return 0;
}

static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
	struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args;
	struct rhashtable_iter *rhiter = &iter->rhiter;
	struct ila_map *ila;
	int ret;

	ret = rhashtable_walk_start(rhiter);
	if (ret && ret != -EAGAIN)
		goto done;

	for (;;) {
		ila = rhashtable_walk_next(rhiter);

		if (IS_ERR(ila)) {
			if (PTR_ERR(ila) == -EAGAIN)
				continue;
			ret = PTR_ERR(ila);
			goto done;
		} else if (!ila) {
			break;
		}

		while (ila) {
			ret =  ila_dump_info(ila, NETLINK_CB(cb->skb).portid,
					     cb->nlh->nlmsg_seq, NLM_F_MULTI,
					     skb, ILA_CMD_GET);
			if (ret)
				goto done;

			ila = rcu_access_pointer(ila->next);
		}
	}

	ret = skb->len;

done:
	rhashtable_walk_stop(rhiter);
	return ret;
}

static const struct genl_ops ila_nl_ops[] = {
	{
		.cmd = ILA_CMD_ADD,
		.doit = ila_nl_cmd_add_mapping,
		.policy = ila_nl_policy,
		.flags = GENL_ADMIN_PERM,
	},
	{
		.cmd = ILA_CMD_DEL,
		.doit = ila_nl_cmd_del_mapping,
		.policy = ila_nl_policy,
		.flags = GENL_ADMIN_PERM,
	},
	{
		.cmd = ILA_CMD_GET,
		.doit = ila_nl_cmd_get_mapping,
		.start = ila_nl_dump_start,
		.dumpit = ila_nl_dump,
		.done = ila_nl_dump_done,
		.policy = ila_nl_policy,
	},
};

#define ILA_HASH_TABLE_SIZE 1024

static __net_init int ila_init_net(struct net *net)
{
	int err;
	struct ila_net *ilan = net_generic(net, ila_net_id);

	err = alloc_ila_locks(ilan);
	if (err)
		return err;

	rhashtable_init(&ilan->rhash_table, &rht_params);

	return 0;
}

static __net_exit void ila_exit_net(struct net *net)
{
	struct ila_net *ilan = net_generic(net, ila_net_id);

	rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL);

	kvfree(ilan->locks);

	if (ilan->hooks_registered)
		nf_unregister_net_hooks(net, ila_nf_hook_ops,
					ARRAY_SIZE(ila_nf_hook_ops));
}

static struct pernet_operations ila_net_ops = {
	.init = ila_init_net,
	.exit = ila_exit_net,
	.id   = &ila_net_id,
	.size = sizeof(struct ila_net),
};

static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral)
{
	struct ila_map *ila;
	struct ipv6hdr *ip6h = ipv6_hdr(skb);
	struct net *net = dev_net(skb->dev);
	struct ila_net *ilan = net_generic(net, ila_net_id);
	struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);

	/* Assumes skb contains a valid IPv6 header that is pulled */

	if (!ila_addr_is_ila(iaddr)) {
		/* Type indicates this is not an ILA address */
		return 0;
	}

	rcu_read_lock();

	ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan);
	if (ila)
		ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral);

	rcu_read_unlock();

	return 0;
}

int ila_xlat_init(void)
{
	int ret;

	ret = register_pernet_device(&ila_net_ops);
	if (ret)
		goto exit;

	ret = genl_register_family_with_ops(&ila_nl_family,
					    ila_nl_ops);
	if (ret < 0)
		goto unregister;

	return 0;

unregister:
	unregister_pernet_device(&ila_net_ops);
exit:
	return ret;
}

void ila_xlat_fini(void)
{
	genl_unregister_family(&ila_nl_family);
	unregister_pernet_device(&ila_net_ops);
}
Commit	Line	Data
7f00feaf TH	1	#include <linux/jhash.h>
	2	#include <linux/netfilter.h>
	3	#include <linux/rcupdate.h>
	4	#include <linux/rhashtable.h>
	5	#include <linux/vmalloc.h>
	6	#include <net/genetlink.h>
	7	#include <net/ila.h>
	8	#include <net/netns/generic.h>
	9	#include <uapi/linux/genetlink.h>
	10	#include "ila.h"
	11
	12	struct ila_xlat_params {
	13	struct ila_params ip;
7f00feaf	14	int ifindex;
7f00feaf TH	15	};
	16
	17	struct ila_map {
351596aa	18	struct ila_xlat_params xp;
7f00feaf TH	19	struct rhash_head node;
	20	struct ila_map __rcu *next;
	21	struct rcu_head rcu;
	22	};
	23
	24	static unsigned int ila_net_id;
	25
	26	struct ila_net {
	27	struct rhashtable rhash_table;
	28	spinlock_t locks; / Bucket locks for entry manipulation */
	29	unsigned int locks_mask;
	30	bool hooks_registered;
	31	};
	32
	33	#define LOCKS_PER_CPU 10
	34
	35	static int alloc_ila_locks(struct ila_net *ilan)
	36	{
	37	unsigned int i, size;
	38	unsigned int nr_pcpus = num_possible_cpus();
	39
	40	nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
	41	size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU);
	42
	43	if (sizeof(spinlock_t) != 0) {
	44	#ifdef CONFIG_NUMA
	45	if (size * sizeof(spinlock_t) > PAGE_SIZE)
	46	ilan->locks = vmalloc(size * sizeof(spinlock_t));
	47	else
	48	#endif
	49	ilan->locks = kmalloc_array(size, sizeof(spinlock_t),
	50	GFP_KERNEL);
	51	if (!ilan->locks)
	52	return -ENOMEM;
	53	for (i = 0; i < size; i++)
	54	spin_lock_init(&ilan->locks[i]);
	55	}
	56	ilan->locks_mask = size - 1;
	57
	58	return 0;
	59	}
	60
	61	static u32 hashrnd __read_mostly;
	62	static __always_inline void __ila_hash_secret_init(void)
	63	{
	64	net_get_random_once(&hashrnd, sizeof(hashrnd));
	65	}
	66
642c2c95	67	static inline u32 ila_locator_hash(struct ila_locator loc)
7f00feaf	68	{
642c2c95	69	u32 v = (u32 )loc.v32;
7f00feaf TH	70
	71	return jhash_2words(v[0], v[1], hashrnd);
	72	}
	73
351596aa	74	static inline spinlock_t ila_get_lock(struct ila_net ilan,
642c2c95	75	struct ila_locator loc)
7f00feaf	76	{
642c2c95	77	return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask];
7f00feaf TH	78	}
7f00feaf TH	79
351596aa	80	static inline int ila_cmp_wildcards(struct ila_map *ila,
642c2c95	81	struct ila_addr *iaddr, int ifindex)
7f00feaf	82	{
642c2c95	83	return (ila->xp.ifindex && ila->xp.ifindex != ifindex);
7f00feaf TH	84	}
7f00feaf TH	85
351596aa TH	86	static inline int ila_cmp_params(struct ila_map *ila,
351596aa TH	87	struct ila_xlat_params *xp)
7f00feaf	88	{
642c2c95	89	return (ila->xp.ifindex != xp->ifindex);
7f00feaf TH	90	}
	91
	92	static int ila_cmpfn(struct rhashtable_compare_arg *arg,
	93	const void *obj)
	94	{
	95	const struct ila_map *ila = obj;
	96
642c2c95	97	return (ila->xp.ip.locator_match.v64 != (__be64 )arg->key);
7f00feaf TH	98	}
	99
	100	static inline int ila_order(struct ila_map *ila)
	101	{
	102	int score = 0;
	103
351596aa	104	if (ila->xp.ifindex)
7f00feaf TH	105	score += 1 << 1;
	106
	107	return score;
	108	}
	109
	110	static const struct rhashtable_params rht_params = {
	111	.nelem_hint = 1024,
	112	.head_offset = offsetof(struct ila_map, node),
642c2c95	113	.key_offset = offsetof(struct ila_map, xp.ip.locator_match),
7f00feaf TH	114	.key_len = sizeof(u64), /* identifier */
	115	.max_size = 1048576,
	116	.min_size = 256,
	117	.automatic_shrinking = true,
	118	.obj_cmpfn = ila_cmpfn,
	119	};
	120
	121	static struct genl_family ila_nl_family = {
	122	.id = GENL_ID_GENERATE,
	123	.hdrsize = 0,
	124	.name = ILA_GENL_NAME,
	125	.version = ILA_GENL_VERSION,
	126	.maxattr = ILA_ATTR_MAX,
	127	.netnsok = true,
	128	.parallel_ops = true,
	129	};
	130
6501f34f	131	static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
7f00feaf TH	132	[ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
	133	[ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
	134	[ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
90bfe662	135	[ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, },
7f00feaf TH	136	};
	137
	138	static int parse_nl_config(struct genl_info *info,
351596aa	139	struct ila_xlat_params *xp)
7f00feaf	140	{
351596aa	141	memset(xp, 0, sizeof(*xp));
7f00feaf	142
7f00feaf	143	if (info->attrs[ILA_ATTR_LOCATOR])
351596aa	144	xp->ip.locator.v64 = (__force __be64)nla_get_u64(
7f00feaf TH	145	info->attrs[ILA_ATTR_LOCATOR]);
	146
	147	if (info->attrs[ILA_ATTR_LOCATOR_MATCH])
351596aa	148	xp->ip.locator_match.v64 = (__force __be64)nla_get_u64(
7f00feaf TH	149	info->attrs[ILA_ATTR_LOCATOR_MATCH]);
7f00feaf TH	150
90bfe662 TH	151	if (info->attrs[ILA_ATTR_CSUM_MODE])
	152	xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]);
	153
7f00feaf	154	if (info->attrs[ILA_ATTR_IFINDEX])
351596aa	155	xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]);
7f00feaf	156
7f00feaf TH	157	return 0;
	158	}
	159
	160	/* Must be called with rcu readlock */
351596aa	161	static inline struct ila_map ila_lookup_wildcards(struct ila_addr iaddr,
7f00feaf	162	int ifindex,
7f00feaf TH	163	struct ila_net *ilan)
	164	{
	165	struct ila_map *ila;
	166
642c2c95	167	ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc,
351596aa	168	rht_params);
7f00feaf	169	while (ila) {
642c2c95	170	if (!ila_cmp_wildcards(ila, iaddr, ifindex))
7f00feaf TH	171	return ila;
	172	ila = rcu_access_pointer(ila->next);
	173	}
	174
	175	return NULL;
	176	}
	177
	178	/* Must be called with rcu readlock */
351596aa	179	static inline struct ila_map ila_lookup_by_params(struct ila_xlat_params xp,
7f00feaf TH	180	struct ila_net *ilan)
	181	{
	182	struct ila_map *ila;
	183
642c2c95 TH	184	ila = rhashtable_lookup_fast(&ilan->rhash_table,
642c2c95 TH	185	&xp->ip.locator_match,
7f00feaf TH	186	rht_params);
7f00feaf TH	187	while (ila) {
351596aa	188	if (!ila_cmp_params(ila, xp))
7f00feaf TH	189	return ila;
	190	ila = rcu_access_pointer(ila->next);
	191	}
	192
	193	return NULL;
	194	}
	195
	196	static inline void ila_release(struct ila_map *ila)
	197	{
	198	kfree_rcu(ila, rcu);
	199	}
	200
	201	static void ila_free_cb(void ptr, void arg)
	202	{
	203	struct ila_map ila = (struct ila_map )ptr, *next;
	204
	205	/* Assume rcu_readlock held */
	206	while (ila) {
	207	next = rcu_access_pointer(ila->next);
	208	ila_release(ila);
	209	ila = next;
	210	}
	211	}
	212
707a2ca4	213	static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral);
7f00feaf TH	214
	215	static unsigned int
	216	ila_nf_input(void *priv,
	217	struct sk_buff *skb,
	218	const struct nf_hook_state *state)
	219	{
707a2ca4	220	ila_xlat_addr(skb, false);
7f00feaf TH	221	return NF_ACCEPT;
	222	}
	223
	224	static struct nf_hook_ops ila_nf_hook_ops[] __read_mostly = {
	225	{
	226	.hook = ila_nf_input,
	227	.pf = NFPROTO_IPV6,
	228	.hooknum = NF_INET_PRE_ROUTING,
	229	.priority = -1,
	230	},
	231	};
	232
351596aa	233	static int ila_add_mapping(struct net net, struct ila_xlat_params xp)
7f00feaf TH	234	{
	235	struct ila_net *ilan = net_generic(net, ila_net_id);
	236	struct ila_map ila, head;
642c2c95	237	spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
7f00feaf TH	238	int err = 0, order;
	239
	240	if (!ilan->hooks_registered) {
	241	/* We defer registering net hooks in the namespace until the
	242	* first mapping is added.
	243	*/
	244	err = nf_register_net_hooks(net, ila_nf_hook_ops,
	245	ARRAY_SIZE(ila_nf_hook_ops));
	246	if (err)
	247	return err;
	248
	249	ilan->hooks_registered = true;
	250	}
	251
	252	ila = kzalloc(sizeof(*ila), GFP_KERNEL);
	253	if (!ila)
	254	return -ENOMEM;
	255
90bfe662	256	ila_init_saved_csum(&xp->ip);
7f00feaf	257
90bfe662	258	ila->xp = *xp;
7f00feaf TH	259
	260	order = ila_order(ila);
	261
	262	spin_lock(lock);
	263
642c2c95 TH	264	head = rhashtable_lookup_fast(&ilan->rhash_table,
642c2c95 TH	265	&xp->ip.locator_match,
7f00feaf TH	266	rht_params);
	267	if (!head) {
	268	/* New entry for the rhash_table */
	269	err = rhashtable_lookup_insert_fast(&ilan->rhash_table,
	270	&ila->node, rht_params);
	271	} else {
	272	struct ila_map tila = head, prev = NULL;
	273
	274	do {
351596aa	275	if (!ila_cmp_params(tila, xp)) {
7f00feaf TH	276	err = -EEXIST;
	277	goto out;
	278	}
	279
	280	if (order > ila_order(tila))
	281	break;
	282
	283	prev = tila;
	284	tila = rcu_dereference_protected(tila->next,
	285	lockdep_is_held(lock));
	286	} while (tila);
	287
	288	if (prev) {
	289	/* Insert in sub list of head */
	290	RCU_INIT_POINTER(ila->next, tila);
	291	rcu_assign_pointer(prev->next, ila);
	292	} else {
	293	/* Make this ila new head */
	294	RCU_INIT_POINTER(ila->next, head);
	295	err = rhashtable_replace_fast(&ilan->rhash_table,
	296	&head->node,
	297	&ila->node, rht_params);
	298	if (err)
	299	goto out;
	300	}
	301	}
	302
	303	out:
	304	spin_unlock(lock);
	305
	306	if (err)
	307	kfree(ila);
	308
	309	return err;
	310	}
	311
351596aa	312	static int ila_del_mapping(struct net net, struct ila_xlat_params xp)
7f00feaf TH	313	{
	314	struct ila_net *ilan = net_generic(net, ila_net_id);
	315	struct ila_map ila, head, *prev;
642c2c95	316	spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
7f00feaf TH	317	int err = -ENOENT;
	318
	319	spin_lock(lock);
	320
	321	head = rhashtable_lookup_fast(&ilan->rhash_table,
642c2c95	322	&xp->ip.locator_match, rht_params);
7f00feaf TH	323	ila = head;
	324
	325	prev = NULL;
	326
	327	while (ila) {
351596aa	328	if (ila_cmp_params(ila, xp)) {
7f00feaf TH	329	prev = ila;
	330	ila = rcu_dereference_protected(ila->next,
	331	lockdep_is_held(lock));
	332	continue;
	333	}
	334
	335	err = 0;
	336
	337	if (prev) {
	338	/* Not head, just delete from list */
	339	rcu_assign_pointer(prev->next, ila->next);
	340	} else {
	341	/* It is the head. If there is something in the
	342	* sublist we need to make a new head.
	343	*/
	344	head = rcu_dereference_protected(ila->next,
	345	lockdep_is_held(lock));
	346	if (head) {
	347	/* Put first entry in the sublist into the
	348	* table
	349	*/
	350	err = rhashtable_replace_fast(
	351	&ilan->rhash_table, &ila->node,
	352	&head->node, rht_params);
	353	if (err)
	354	goto out;
	355	} else {
	356	/* Entry no longer used */
	357	err = rhashtable_remove_fast(&ilan->rhash_table,
	358	&ila->node,
	359	rht_params);
	360	}
	361	}
	362
	363	ila_release(ila);
	364
	365	break;
	366	}
	367
	368	out:
	369	spin_unlock(lock);
	370
	371	return err;
	372	}
	373
	374	static int ila_nl_cmd_add_mapping(struct sk_buff skb, struct genl_info info)
	375	{
	376	struct net *net = genl_info_net(info);
	377	struct ila_xlat_params p;
	378	int err;
	379
	380	err = parse_nl_config(info, &p);
	381	if (err)
	382	return err;
	383
	384	return ila_add_mapping(net, &p);
	385	}
	386
	387	static int ila_nl_cmd_del_mapping(struct sk_buff skb, struct genl_info info)
	388	{
	389	struct net *net = genl_info_net(info);
351596aa	390	struct ila_xlat_params xp;
7f00feaf TH	391	int err;
7f00feaf TH	392
351596aa	393	err = parse_nl_config(info, &xp);
7f00feaf TH	394	if (err)
	395	return err;
	396
351596aa	397	ila_del_mapping(net, &xp);
7f00feaf TH	398
	399	return 0;
	400	}
	401
	402	static int ila_fill_info(struct ila_map ila, struct sk_buff msg)
	403	{
642c2c95	404	if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR,
351596aa	405	(__force u64)ila->xp.ip.locator.v64,
f13a82d8 ND	406	ILA_ATTR_PAD) \|\|
f13a82d8 ND	407	nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH,
351596aa	408	(__force u64)ila->xp.ip.locator_match.v64,
f13a82d8	409	ILA_ATTR_PAD) \|\|
90bfe662 TH	410	nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) \|\|
90bfe662 TH	411	nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode))
7f00feaf TH	412	return -1;
	413
	414	return 0;
	415	}
	416
	417	static int ila_dump_info(struct ila_map *ila,
	418	u32 portid, u32 seq, u32 flags,
	419	struct sk_buff *skb, u8 cmd)
	420	{
	421	void *hdr;
	422
	423	hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd);
	424	if (!hdr)
	425	return -ENOMEM;
	426
	427	if (ila_fill_info(ila, skb) < 0)
	428	goto nla_put_failure;
	429
	430	genlmsg_end(skb, hdr);
	431	return 0;
	432
	433	nla_put_failure:
	434	genlmsg_cancel(skb, hdr);
	435	return -EMSGSIZE;
	436	}
	437
	438	static int ila_nl_cmd_get_mapping(struct sk_buff skb, struct genl_info info)
	439	{
	440	struct net *net = genl_info_net(info);
	441	struct ila_net *ilan = net_generic(net, ila_net_id);
	442	struct sk_buff *msg;
351596aa	443	struct ila_xlat_params xp;
7f00feaf TH	444	struct ila_map *ila;
	445	int ret;
	446
351596aa	447	ret = parse_nl_config(info, &xp);
7f00feaf TH	448	if (ret)
	449	return ret;
	450
	451	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
	452	if (!msg)
	453	return -ENOMEM;
	454
	455	rcu_read_lock();
	456
351596aa	457	ila = ila_lookup_by_params(&xp, ilan);
7f00feaf TH	458	if (ila) {
	459	ret = ila_dump_info(ila,
	460	info->snd_portid,
	461	info->snd_seq, 0, msg,
	462	info->genlhdr->cmd);
	463	}
	464
	465	rcu_read_unlock();
	466
	467	if (ret < 0)
	468	goto out_free;
	469
	470	return genlmsg_reply(msg, info);
	471
	472	out_free:
	473	nlmsg_free(msg);
	474	return ret;
	475	}
	476
	477	struct ila_dump_iter {
	478	struct rhashtable_iter rhiter;
	479	};
	480
	481	static int ila_nl_dump_start(struct netlink_callback *cb)
	482	{
	483	struct net *net = sock_net(cb->skb->sk);
	484	struct ila_net *ilan = net_generic(net, ila_net_id);
	485	struct ila_dump_iter iter = (struct ila_dump_iter )cb->args;
	486
8f6fd83c BC	487	return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter,
8f6fd83c BC	488	GFP_KERNEL);
7f00feaf TH	489	}
	490
	491	static int ila_nl_dump_done(struct netlink_callback *cb)
	492	{
	493	struct ila_dump_iter iter = (struct ila_dump_iter )cb->args;
	494
	495	rhashtable_walk_exit(&iter->rhiter);
	496
	497	return 0;
	498	}
	499
	500	static int ila_nl_dump(struct sk_buff skb, struct netlink_callback cb)
	501	{
	502	struct ila_dump_iter iter = (struct ila_dump_iter )cb->args;
	503	struct rhashtable_iter *rhiter = &iter->rhiter;
	504	struct ila_map *ila;
	505	int ret;
	506
	507	ret = rhashtable_walk_start(rhiter);
	508	if (ret && ret != -EAGAIN)
	509	goto done;
	510
	511	for (;;) {
	512	ila = rhashtable_walk_next(rhiter);
	513
	514	if (IS_ERR(ila)) {
	515	if (PTR_ERR(ila) == -EAGAIN)
	516	continue;
	517	ret = PTR_ERR(ila);
	518	goto done;
	519	} else if (!ila) {
	520	break;
	521	}
	522
	523	while (ila) {
	524	ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid,
	525	cb->nlh->nlmsg_seq, NLM_F_MULTI,
	526	skb, ILA_CMD_GET);
	527	if (ret)
	528	goto done;
	529
	530	ila = rcu_access_pointer(ila->next);
	531	}
	532	}
	533
	534	ret = skb->len;
	535
	536	done:
	537	rhashtable_walk_stop(rhiter);
	538	return ret;
	539	}
	540
	541	static const struct genl_ops ila_nl_ops[] = {
	542	{
	543	.cmd = ILA_CMD_ADD,
	544	.doit = ila_nl_cmd_add_mapping,
	545	.policy = ila_nl_policy,
	546	.flags = GENL_ADMIN_PERM,
	547	},
	548	{
	549	.cmd = ILA_CMD_DEL,
	550	.doit = ila_nl_cmd_del_mapping,
	551	.policy = ila_nl_policy,
	552	.flags = GENL_ADMIN_PERM,
553	},
554	{
555	.cmd = ILA_CMD_GET,
556	.doit = ila_nl_cmd_get_mapping,
557	.start = ila_nl_dump_start,
558	.dumpit = ila_nl_dump,
559	.done = ila_nl_dump_done,
560	.policy = ila_nl_policy,
561	},
562	};
563
564	#define ILA_HASH_TABLE_SIZE 1024
565
566	static __net_init int ila_init_net(struct net *net)
567	{
568	int err;
569	struct ila_net *ilan = net_generic(net, ila_net_id);
570
571	err = alloc_ila_locks(ilan);
572	if (err)
573	return err;
574
575	rhashtable_init(&ilan->rhash_table, &rht_params);
576
577	return 0;
578	}
579
580	static __net_exit void ila_exit_net(struct net *net)
581	{
582	struct ila_net *ilan = net_generic(net, ila_net_id);
583
584	rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL);
585
586	kvfree(ilan->locks);
587
588	if (ilan->hooks_registered)
589	nf_unregister_net_hooks(net, ila_nf_hook_ops,
590	ARRAY_SIZE(ila_nf_hook_ops));
591	}
592
593	static struct pernet_operations ila_net_ops = {
594	.init = ila_init_net,
595	.exit = ila_exit_net,
596	.id = &ila_net_id,
597	.size = sizeof(struct ila_net),
598	};
599
707a2ca4	600	static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral)
7f00feaf TH	601	{
	602	struct ila_map *ila;
	603	struct ipv6hdr *ip6h = ipv6_hdr(skb);
	604	struct net *net = dev_net(skb->dev);
	605	struct ila_net *ilan = net_generic(net, ila_net_id);
351596aa	606	struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);
7f00feaf TH	607
	608	/* Assumes skb contains a valid IPv6 header that is pulled */
	609
642c2c95 TH	610	if (!ila_addr_is_ila(iaddr)) {
	611	/* Type indicates this is not an ILA address */
	612	return 0;
	613	}
7f00feaf TH	614
	615	rcu_read_lock();
	616
642c2c95	617	ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan);
7f00feaf	618	if (ila)
707a2ca4	619	ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral);
7f00feaf TH	620
	621	rcu_read_unlock();
	622
	623	return 0;
	624	}
	625
7f00feaf TH	626	int ila_xlat_init(void)
	627	{
	628	int ret;
	629
	630	ret = register_pernet_device(&ila_net_ops);
	631	if (ret)
	632	goto exit;
	633
	634	ret = genl_register_family_with_ops(&ila_nl_family,
	635	ila_nl_ops);
	636	if (ret < 0)
	637	goto unregister;
	638
	639	return 0;
	640
	641	unregister:
	642	unregister_pernet_device(&ila_net_ops);
	643	exit:
	644	return ret;
	645	}
	646
	647	void ila_xlat_fini(void)
	648	{
	649	genl_unregister_family(&ila_nl_family);
	650	unregister_pernet_device(&ila_net_ops);
	651	}