[deliverable/linux.git] / net / ipv4 / gre_offload.c

/*
 *	IPV4 GSO/GRO offload support
 *	Linux INET implementation
 *
 *	This program is free software; you can redistribute it and/or
 *	modify it under the terms of the GNU General Public License
 *	as published by the Free Software Foundation; either version
 *	2 of the License, or (at your option) any later version.
 *
 *	GRE GSO support
 */

#include <linux/skbuff.h>
#include <linux/init.h>
#include <net/protocol.h>
#include <net/gre.h>

static int gre_gso_send_check(struct sk_buff *skb)
{
	if (!skb->encapsulation)
		return -EINVAL;
	return 0;
}

static struct sk_buff *gre_gso_segment(struct sk_buff *skb,
				       netdev_features_t features)
{
	struct sk_buff *segs = ERR_PTR(-EINVAL);
	netdev_features_t enc_features;
	int ghl;
	struct gre_base_hdr *greh;
	u16 mac_offset = skb->mac_header;
	int mac_len = skb->mac_len;
	__be16 protocol = skb->protocol;
	int tnl_hlen;
	bool csum;

	if (unlikely(skb_shinfo(skb)->gso_type &
				~(SKB_GSO_TCPV4 |
				  SKB_GSO_TCPV6 |
				  SKB_GSO_UDP |
				  SKB_GSO_DODGY |
				  SKB_GSO_TCP_ECN |
				  SKB_GSO_GRE |
				  SKB_GSO_GRE_CSUM |
				  SKB_GSO_IPIP)))
		goto out;

	if (unlikely(!pskb_may_pull(skb, sizeof(*greh))))
		goto out;

	greh = (struct gre_base_hdr *)skb_transport_header(skb);

	ghl = skb_inner_network_header(skb) - skb_transport_header(skb);
	if (unlikely(ghl < sizeof(*greh)))
		goto out;

	csum = !!(greh->flags & GRE_CSUM);
	if (csum)
		skb->encap_hdr_csum = 1;

	if (unlikely(!pskb_may_pull(skb, ghl)))
		goto out;

	/* setup inner skb. */
	skb->protocol = greh->protocol;
	skb->encapsulation = 0;

	__skb_pull(skb, ghl);
	skb_reset_mac_header(skb);
	skb_set_network_header(skb, skb_inner_network_offset(skb));
	skb->mac_len = skb_inner_network_offset(skb);

	/* segment inner packet. */
	enc_features = skb->dev->hw_enc_features & netif_skb_features(skb);
	segs = skb_mac_gso_segment(skb, enc_features);
	if (IS_ERR_OR_NULL(segs)) {
		skb_gso_error_unwind(skb, protocol, ghl, mac_offset, mac_len);
		goto out;
	}

	skb = segs;
	tnl_hlen = skb_tnl_header_len(skb);
	do {
		__skb_push(skb, ghl);
		if (csum) {
			__be32 *pcsum;

			if (skb_has_shared_frag(skb)) {
				int err;

				err = __skb_linearize(skb);
				if (err) {
					kfree_skb_list(segs);
					segs = ERR_PTR(err);
					goto out;
				}
			}

			skb_reset_transport_header(skb);

			greh = (struct gre_base_hdr *)
			    skb_transport_header(skb);
			pcsum = (__be32 *)(greh + 1);
			*pcsum = 0;
			*(__sum16 *)pcsum = gso_make_checksum(skb, 0);
		}
		__skb_push(skb, tnl_hlen - ghl);

		skb_reset_inner_headers(skb);
		skb->encapsulation = 1;

		skb_reset_mac_header(skb);
		skb_set_network_header(skb, mac_len);
		skb->mac_len = mac_len;
		skb->protocol = protocol;
	} while ((skb = skb->next));
out:
	return segs;
}

static struct sk_buff **gre_gro_receive(struct sk_buff **head,
					struct sk_buff *skb)
{
	struct sk_buff **pp = NULL;
	struct sk_buff *p;
	const struct gre_base_hdr *greh;
	unsigned int hlen, grehlen;
	unsigned int off;
	int flush = 1;
	struct packet_offload *ptype;
	__be16 type;

	off = skb_gro_offset(skb);
	hlen = off + sizeof(*greh);
	greh = skb_gro_header_fast(skb, off);
	if (skb_gro_header_hard(skb, hlen)) {
		greh = skb_gro_header_slow(skb, hlen, off);
		if (unlikely(!greh))
			goto out;
	}

	/* Only support version 0 and K (key), C (csum) flags. Note that
	 * although the support for the S (seq#) flag can be added easily
	 * for GRO, this is problematic for GSO hence can not be enabled
	 * here because a GRO pkt may end up in the forwarding path, thus
	 * requiring GSO support to break it up correctly.
	 */
	if ((greh->flags & ~(GRE_KEY|GRE_CSUM)) != 0)
		goto out;

	type = greh->protocol;

	rcu_read_lock();
	ptype = gro_find_receive_by_type(type);
	if (ptype == NULL)
		goto out_unlock;

	grehlen = GRE_HEADER_SECTION;

	if (greh->flags & GRE_KEY)
		grehlen += GRE_HEADER_SECTION;

	if (greh->flags & GRE_CSUM)
		grehlen += GRE_HEADER_SECTION;

	hlen = off + grehlen;
	if (skb_gro_header_hard(skb, hlen)) {
		greh = skb_gro_header_slow(skb, hlen, off);
		if (unlikely(!greh))
			goto out_unlock;
	}

	/* Don't bother verifying checksum if we're going to flush anyway. */
	if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush &&
	    skb_gro_checksum_simple_validate(skb))
			goto out_unlock;

	flush = 0;

	for (p = *head; p; p = p->next) {
		const struct gre_base_hdr *greh2;

		if (!NAPI_GRO_CB(p)->same_flow)
			continue;

		/* The following checks are needed to ensure only pkts
		 * from the same tunnel are considered for aggregation.
		 * The criteria for "the same tunnel" includes:
		 * 1) same version (we only support version 0 here)
		 * 2) same protocol (we only support ETH_P_IP for now)
		 * 3) same set of flags
		 * 4) same key if the key field is present.
		 */
		greh2 = (struct gre_base_hdr *)(p->data + off);

		if (greh2->flags != greh->flags ||
		    greh2->protocol != greh->protocol) {
			NAPI_GRO_CB(p)->same_flow = 0;
			continue;
		}
		if (greh->flags & GRE_KEY) {
			/* compare keys */
			if (*(__be32 *)(greh2+1) != *(__be32 *)(greh+1)) {
				NAPI_GRO_CB(p)->same_flow = 0;
				continue;
			}
		}
	}

	skb_gro_pull(skb, grehlen);

	/* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/
	skb_gro_postpull_rcsum(skb, greh, grehlen);

	pp = ptype->callbacks.gro_receive(head, skb);

out_unlock:
	rcu_read_unlock();
out:
	NAPI_GRO_CB(skb)->flush |= flush;

	return pp;
}

static int gre_gro_complete(struct sk_buff *skb, int nhoff)
{
	struct gre_base_hdr *greh = (struct gre_base_hdr *)(skb->data + nhoff);
	struct packet_offload *ptype;
	unsigned int grehlen = sizeof(*greh);
	int err = -ENOENT;
	__be16 type;

	skb->encapsulation = 1;
	skb_shinfo(skb)->gso_type = SKB_GSO_GRE;

	type = greh->protocol;
	if (greh->flags & GRE_KEY)
		grehlen += GRE_HEADER_SECTION;

	if (greh->flags & GRE_CSUM)
		grehlen += GRE_HEADER_SECTION;

	rcu_read_lock();
	ptype = gro_find_complete_by_type(type);
	if (ptype != NULL)
		err = ptype->callbacks.gro_complete(skb, nhoff + grehlen);

	rcu_read_unlock();
	return err;
}

static const struct net_offload gre_offload = {
	.callbacks = {
		.gso_send_check = gre_gso_send_check,
		.gso_segment = gre_gso_segment,
		.gro_receive = gre_gro_receive,
		.gro_complete = gre_gro_complete,
	},
};

static int __init gre_offload_init(void)
{
	return inet_add_offload(&gre_offload, IPPROTO_GRE);
}
device_initcall(gre_offload_init);
Commit	Line	Data
c50cd357 DB	1	/*
	2	* IPV4 GSO/GRO offload support
	3	* Linux INET implementation
	4	*
	5	* This program is free software; you can redistribute it and/or
	6	* modify it under the terms of the GNU General Public License
	7	* as published by the Free Software Foundation; either version
	8	* 2 of the License, or (at your option) any later version.
	9	*
	10	* GRE GSO support
	11	*/
	12
	13	#include <linux/skbuff.h>
cf172283	14	#include <linux/init.h>
c50cd357 DB	15	#include <net/protocol.h>
	16	#include <net/gre.h>
	17
	18	static int gre_gso_send_check(struct sk_buff *skb)
	19	{
	20	if (!skb->encapsulation)
	21	return -EINVAL;
	22	return 0;
	23	}
	24
	25	static struct sk_buff gre_gso_segment(struct sk_buff skb,
	26	netdev_features_t features)
	27	{
	28	struct sk_buff *segs = ERR_PTR(-EINVAL);
	29	netdev_features_t enc_features;
b884b1a4	30	int ghl;
c50cd357	31	struct gre_base_hdr *greh;
7a7ffbab	32	u16 mac_offset = skb->mac_header;
c50cd357 DB	33	int mac_len = skb->mac_len;
	34	__be16 protocol = skb->protocol;
	35	int tnl_hlen;
	36	bool csum;
	37
	38	if (unlikely(skb_shinfo(skb)->gso_type &
	39	~(SKB_GSO_TCPV4 \|
	40	SKB_GSO_TCPV6 \|
	41	SKB_GSO_UDP \|
	42	SKB_GSO_DODGY \|
	43	SKB_GSO_TCP_ECN \|
cb32f511	44	SKB_GSO_GRE \|
4749c09c	45	SKB_GSO_GRE_CSUM \|
cb32f511	46	SKB_GSO_IPIP)))
c50cd357 DB	47	goto out;
	48
	49	if (unlikely(!pskb_may_pull(skb, sizeof(*greh))))
	50	goto out;
	51
	52	greh = (struct gre_base_hdr *)skb_transport_header(skb);
	53
b884b1a4 NC	54	ghl = skb_inner_network_header(skb) - skb_transport_header(skb);
	55	if (unlikely(ghl < sizeof(*greh)))
	56	goto out;
	57
	58	csum = !!(greh->flags & GRE_CSUM);
4749c09c TH	59	if (csum)
4749c09c TH	60	skb->encap_hdr_csum = 1;
c50cd357	61
7a7ffbab WCC	62	if (unlikely(!pskb_may_pull(skb, ghl)))
	63	goto out;
	64
c50cd357 DB	65	/* setup inner skb. */
	66	skb->protocol = greh->protocol;
	67	skb->encapsulation = 0;
	68
c50cd357 DB	69	__skb_pull(skb, ghl);
	70	skb_reset_mac_header(skb);
	71	skb_set_network_header(skb, skb_inner_network_offset(skb));
	72	skb->mac_len = skb_inner_network_offset(skb);
	73
	74	/* segment inner packet. */
	75	enc_features = skb->dev->hw_enc_features & netif_skb_features(skb);
	76	segs = skb_mac_gso_segment(skb, enc_features);
5a8dbf03	77	if (IS_ERR_OR_NULL(segs)) {
7a7ffbab	78	skb_gso_error_unwind(skb, protocol, ghl, mac_offset, mac_len);
c50cd357	79	goto out;
7a7ffbab	80	}
c50cd357 DB	81
	82	skb = segs;
	83	tnl_hlen = skb_tnl_header_len(skb);
	84	do {
	85	__skb_push(skb, ghl);
	86	if (csum) {
	87	__be32 *pcsum;
	88
	89	if (skb_has_shared_frag(skb)) {
	90	int err;
	91
	92	err = __skb_linearize(skb);
	93	if (err) {
0c1072ae	94	kfree_skb_list(segs);
c50cd357 DB	95	segs = ERR_PTR(err);
	96	goto out;
	97	}
	98	}
	99
4749c09c TH	100	skb_reset_transport_header(skb);
	101
	102	greh = (struct gre_base_hdr *)
	103	skb_transport_header(skb);
c50cd357 DB	104	pcsum = (__be32 *)(greh + 1);
c50cd357 DB	105	*pcsum = 0;
4749c09c	106	(__sum16 )pcsum = gso_make_checksum(skb, 0);
c50cd357 DB	107	}
	108	__skb_push(skb, tnl_hlen - ghl);
	109
cdbaa0bb AD	110	skb_reset_inner_headers(skb);
	111	skb->encapsulation = 1;
	112
c50cd357 DB	113	skb_reset_mac_header(skb);
	114	skb_set_network_header(skb, mac_len);
	115	skb->mac_len = mac_len;
	116	skb->protocol = protocol;
	117	} while ((skb = skb->next));
	118	out:
	119	return segs;
	120	}
	121
bf5a755f JC	122	static struct sk_buff gre_gro_receive(struct sk_buff head,
	123	struct sk_buff *skb)
	124	{
	125	struct sk_buff **pp = NULL;
	126	struct sk_buff *p;
	127	const struct gre_base_hdr *greh;
	128	unsigned int hlen, grehlen;
	129	unsigned int off;
	130	int flush = 1;
	131	struct packet_offload *ptype;
	132	__be16 type;
	133
	134	off = skb_gro_offset(skb);
	135	hlen = off + sizeof(*greh);
	136	greh = skb_gro_header_fast(skb, off);
	137	if (skb_gro_header_hard(skb, hlen)) {
	138	greh = skb_gro_header_slow(skb, hlen, off);
	139	if (unlikely(!greh))
	140	goto out;
	141	}
	142
	143	/* Only support version 0 and K (key), C (csum) flags. Note that
	144	* although the support for the S (seq#) flag can be added easily
	145	* for GRO, this is problematic for GSO hence can not be enabled
	146	* here because a GRO pkt may end up in the forwarding path, thus
	147	* requiring GSO support to break it up correctly.
	148	*/
	149	if ((greh->flags & ~(GRE_KEY\|GRE_CSUM)) != 0)
	150	goto out;
	151
	152	type = greh->protocol;
	153
	154	rcu_read_lock();
	155	ptype = gro_find_receive_by_type(type);
	156	if (ptype == NULL)
	157	goto out_unlock;
	158
	159	grehlen = GRE_HEADER_SECTION;
	160
	161	if (greh->flags & GRE_KEY)
	162	grehlen += GRE_HEADER_SECTION;
	163
	164	if (greh->flags & GRE_CSUM)
	165	grehlen += GRE_HEADER_SECTION;
	166
	167	hlen = off + grehlen;
	168	if (skb_gro_header_hard(skb, hlen)) {
	169	greh = skb_gro_header_slow(skb, hlen, off);
	170	if (unlikely(!greh))
	171	goto out_unlock;
	172	}
758f75d1 TH	173
758f75d1 TH	174	/* Don't bother verifying checksum if we're going to flush anyway. */
662880f4 TH	175	if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush &&
662880f4 TH	176	skb_gro_checksum_simple_validate(skb))
bf5a755f	177	goto out_unlock;
758f75d1	178
bf5a755f JC	179	flush = 0;
	180
	181	for (p = *head; p; p = p->next) {
	182	const struct gre_base_hdr *greh2;
	183
	184	if (!NAPI_GRO_CB(p)->same_flow)
	185	continue;
	186
	187	/* The following checks are needed to ensure only pkts
	188	* from the same tunnel are considered for aggregation.
	189	* The criteria for "the same tunnel" includes:
	190	* 1) same version (we only support version 0 here)
	191	* 2) same protocol (we only support ETH_P_IP for now)
	192	* 3) same set of flags
	193	* 4) same key if the key field is present.
	194	*/
	195	greh2 = (struct gre_base_hdr *)(p->data + off);
	196
	197	if (greh2->flags != greh->flags \|\|
	198	greh2->protocol != greh->protocol) {
	199	NAPI_GRO_CB(p)->same_flow = 0;
	200	continue;
	201	}
	202	if (greh->flags & GRE_KEY) {
	203	/* compare keys */
	204	if ((__be32 )(greh2+1) != (__be32 )(greh+1)) {
	205	NAPI_GRO_CB(p)->same_flow = 0;
	206	continue;
	207	}
	208	}
	209	}
	210
	211	skb_gro_pull(skb, grehlen);
	212
	213	/* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/
	214	skb_gro_postpull_rcsum(skb, greh, grehlen);
	215
	216	pp = ptype->callbacks.gro_receive(head, skb);
	217
	218	out_unlock:
	219	rcu_read_unlock();
	220	out:
	221	NAPI_GRO_CB(skb)->flush \|= flush;
	222
	223	return pp;
	224	}
	225
d10dbad2	226	static int gre_gro_complete(struct sk_buff *skb, int nhoff)
bf5a755f JC	227	{
	228	struct gre_base_hdr greh = (struct gre_base_hdr )(skb->data + nhoff);
	229	struct packet_offload *ptype;
	230	unsigned int grehlen = sizeof(*greh);
	231	int err = -ENOENT;
	232	__be16 type;
	233
c3caf119 JC	234	skb->encapsulation = 1;
	235	skb_shinfo(skb)->gso_type = SKB_GSO_GRE;
	236
bf5a755f JC	237	type = greh->protocol;
	238	if (greh->flags & GRE_KEY)
	239	grehlen += GRE_HEADER_SECTION;
	240
	241	if (greh->flags & GRE_CSUM)
	242	grehlen += GRE_HEADER_SECTION;
	243
	244	rcu_read_lock();
	245	ptype = gro_find_complete_by_type(type);
	246	if (ptype != NULL)
	247	err = ptype->callbacks.gro_complete(skb, nhoff + grehlen);
	248
	249	rcu_read_unlock();
	250	return err;
	251	}
	252
c50cd357 DB	253	static const struct net_offload gre_offload = {
	254	.callbacks = {
	255	.gso_send_check = gre_gso_send_check,
	256	.gso_segment = gre_gso_segment,
bf5a755f JC	257	.gro_receive = gre_gro_receive,
bf5a755f JC	258	.gro_complete = gre_gro_complete,
c50cd357 DB	259	},
	260	};
	261
438e38fa	262	static int __init gre_offload_init(void)
c50cd357 DB	263	{
	264	return inet_add_offload(&gre_offload, IPPROTO_GRE);
	265	}
cf172283	266	device_initcall(gre_offload_init);