2 * IPV6 GSO/GRO offload support
3 * Linux INET6 implementation
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version
8 * 2 of the License, or (at your option) any later version.
11 #include <linux/kernel.h>
12 #include <linux/socket.h>
13 #include <linux/netdevice.h>
14 #include <linux/skbuff.h>
15 #include <linux/printk.h>
17 #include <net/protocol.h>
20 #include "ip6_offload.h"
22 static int ipv6_gso_pull_exthdrs(struct sk_buff
*skb
, int proto
)
24 const struct net_offload
*ops
= NULL
;
27 struct ipv6_opt_hdr
*opth
;
30 if (proto
!= NEXTHDR_HOP
) {
31 ops
= rcu_dereference(inet6_offloads
[proto
]);
36 if (!(ops
->flags
& INET6_PROTO_GSO_EXTHDR
))
40 if (unlikely(!pskb_may_pull(skb
, 8)))
43 opth
= (void *)skb
->data
;
44 len
= ipv6_optlen(opth
);
46 if (unlikely(!pskb_may_pull(skb
, len
)))
49 proto
= opth
->nexthdr
;
56 static int ipv6_gso_send_check(struct sk_buff
*skb
)
58 const struct ipv6hdr
*ipv6h
;
59 const struct net_offload
*ops
;
62 if (unlikely(!pskb_may_pull(skb
, sizeof(*ipv6h
))))
65 ipv6h
= ipv6_hdr(skb
);
66 __skb_pull(skb
, sizeof(*ipv6h
));
67 err
= -EPROTONOSUPPORT
;
69 ops
= rcu_dereference(inet6_offloads
[
70 ipv6_gso_pull_exthdrs(skb
, ipv6h
->nexthdr
)]);
72 if (likely(ops
&& ops
->callbacks
.gso_send_check
)) {
73 skb_reset_transport_header(skb
);
74 err
= ops
->callbacks
.gso_send_check(skb
);
81 static struct sk_buff
*ipv6_gso_segment(struct sk_buff
*skb
,
82 netdev_features_t features
)
84 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
85 struct ipv6hdr
*ipv6h
;
86 const struct net_offload
*ops
;
88 struct frag_hdr
*fptr
;
89 unsigned int unfrag_ip6hlen
;
95 if (unlikely(skb_shinfo(skb
)->gso_type
&
104 SKB_GSO_UDP_TUNNEL_CSUM
|
110 skb_reset_network_header(skb
);
111 nhoff
= skb_network_header(skb
) - skb_mac_header(skb
);
112 if (unlikely(!pskb_may_pull(skb
, sizeof(*ipv6h
))))
115 encap
= SKB_GSO_CB(skb
)->encap_level
> 0;
117 features
= skb
->dev
->hw_enc_features
& netif_skb_features(skb
);
118 SKB_GSO_CB(skb
)->encap_level
+= sizeof(*ipv6h
);
120 ipv6h
= ipv6_hdr(skb
);
121 __skb_pull(skb
, sizeof(*ipv6h
));
122 segs
= ERR_PTR(-EPROTONOSUPPORT
);
124 proto
= ipv6_gso_pull_exthdrs(skb
, ipv6h
->nexthdr
);
126 if (skb
->encapsulation
&&
127 skb_shinfo(skb
)->gso_type
& (SKB_GSO_SIT
|SKB_GSO_IPIP
))
128 udpfrag
= proto
== IPPROTO_UDP
&& encap
;
130 udpfrag
= proto
== IPPROTO_UDP
&& !skb
->encapsulation
;
132 ops
= rcu_dereference(inet6_offloads
[proto
]);
133 if (likely(ops
&& ops
->callbacks
.gso_segment
)) {
134 skb_reset_transport_header(skb
);
135 segs
= ops
->callbacks
.gso_segment(skb
, features
);
141 for (skb
= segs
; skb
; skb
= skb
->next
) {
142 ipv6h
= (struct ipv6hdr
*)(skb_mac_header(skb
) + nhoff
);
143 ipv6h
->payload_len
= htons(skb
->len
- nhoff
- sizeof(*ipv6h
));
144 skb
->network_header
= (u8
*)ipv6h
- skb
->head
;
147 unfrag_ip6hlen
= ip6_find_1stfragopt(skb
, &prevhdr
);
148 fptr
= (struct frag_hdr
*)((u8
*)ipv6h
+ unfrag_ip6hlen
);
149 fptr
->frag_off
= htons(offset
);
150 if (skb
->next
!= NULL
)
151 fptr
->frag_off
|= htons(IP6_MF
);
152 offset
+= (ntohs(ipv6h
->payload_len
) -
153 sizeof(struct frag_hdr
));
156 skb_reset_inner_headers(skb
);
163 /* Return the total length of all the extension hdrs, following the same
164 * logic in ipv6_gso_pull_exthdrs() when parsing ext-hdrs.
166 static int ipv6_exthdrs_len(struct ipv6hdr
*iph
,
167 const struct net_offload
**opps
)
169 struct ipv6_opt_hdr
*opth
= (void *)iph
;
170 int len
= 0, proto
, optlen
= sizeof(*iph
);
172 proto
= iph
->nexthdr
;
174 if (proto
!= NEXTHDR_HOP
) {
175 *opps
= rcu_dereference(inet6_offloads
[proto
]);
176 if (unlikely(!(*opps
)))
178 if (!((*opps
)->flags
& INET6_PROTO_GSO_EXTHDR
))
181 opth
= (void *)opth
+ optlen
;
182 optlen
= ipv6_optlen(opth
);
184 proto
= opth
->nexthdr
;
189 static struct sk_buff
**ipv6_gro_receive(struct sk_buff
**head
,
192 const struct net_offload
*ops
;
193 struct sk_buff
**pp
= NULL
;
202 off
= skb_gro_offset(skb
);
203 hlen
= off
+ sizeof(*iph
);
204 iph
= skb_gro_header_fast(skb
, off
);
205 if (skb_gro_header_hard(skb
, hlen
)) {
206 iph
= skb_gro_header_slow(skb
, hlen
, off
);
211 skb_set_network_header(skb
, off
);
212 skb_gro_pull(skb
, sizeof(*iph
));
213 skb_set_transport_header(skb
, skb_gro_offset(skb
));
215 flush
+= ntohs(iph
->payload_len
) != skb_gro_len(skb
);
218 proto
= iph
->nexthdr
;
219 ops
= rcu_dereference(inet6_offloads
[proto
]);
220 if (!ops
|| !ops
->callbacks
.gro_receive
) {
221 __pskb_pull(skb
, skb_gro_offset(skb
));
222 proto
= ipv6_gso_pull_exthdrs(skb
, proto
);
223 skb_gro_pull(skb
, -skb_transport_offset(skb
));
224 skb_reset_transport_header(skb
);
225 __skb_push(skb
, skb_gro_offset(skb
));
227 ops
= rcu_dereference(inet6_offloads
[proto
]);
228 if (!ops
|| !ops
->callbacks
.gro_receive
)
234 NAPI_GRO_CB(skb
)->proto
= proto
;
237 nlen
= skb_network_header_len(skb
);
239 for (p
= *head
; p
; p
= p
->next
) {
240 const struct ipv6hdr
*iph2
;
241 __be32 first_word
; /* <Version:4><Traffic_Class:8><Flow_Label:20> */
243 if (!NAPI_GRO_CB(p
)->same_flow
)
246 iph2
= (struct ipv6hdr
*)(p
->data
+ off
);
247 first_word
= *(__be32
*)iph
^ *(__be32
*)iph2
;
249 /* All fields must match except length and Traffic Class.
250 * XXX skbs on the gro_list have all been parsed and pulled
251 * already so we don't need to compare nlen
252 * (nlen != (sizeof(*iph2) + ipv6_exthdrs_len(iph2, &ops)))
253 * memcmp() alone below is suffcient, right?
255 if ((first_word
& htonl(0xF00FFFFF)) ||
256 memcmp(&iph
->nexthdr
, &iph2
->nexthdr
,
257 nlen
- offsetof(struct ipv6hdr
, nexthdr
))) {
258 NAPI_GRO_CB(p
)->same_flow
= 0;
261 /* flush if Traffic Class fields are different */
262 NAPI_GRO_CB(p
)->flush
|= !!(first_word
& htonl(0x0FF00000));
263 NAPI_GRO_CB(p
)->flush
|= flush
;
266 NAPI_GRO_CB(skb
)->flush
|= flush
;
268 skb_gro_postpull_rcsum(skb
, iph
, nlen
);
270 pp
= ops
->callbacks
.gro_receive(head
, skb
);
276 NAPI_GRO_CB(skb
)->flush
|= flush
;
281 static int ipv6_gro_complete(struct sk_buff
*skb
, int nhoff
)
283 const struct net_offload
*ops
;
284 struct ipv6hdr
*iph
= (struct ipv6hdr
*)(skb
->data
+ nhoff
);
287 iph
->payload_len
= htons(skb
->len
- nhoff
- sizeof(*iph
));
291 nhoff
+= sizeof(*iph
) + ipv6_exthdrs_len(iph
, &ops
);
292 if (WARN_ON(!ops
|| !ops
->callbacks
.gro_complete
))
295 err
= ops
->callbacks
.gro_complete(skb
, nhoff
);
303 static struct packet_offload ipv6_packet_offload __read_mostly
= {
304 .type
= cpu_to_be16(ETH_P_IPV6
),
306 .gso_send_check
= ipv6_gso_send_check
,
307 .gso_segment
= ipv6_gso_segment
,
308 .gro_receive
= ipv6_gro_receive
,
309 .gro_complete
= ipv6_gro_complete
,
313 static const struct net_offload sit_offload
= {
315 .gso_send_check
= ipv6_gso_send_check
,
316 .gso_segment
= ipv6_gso_segment
,
320 static int __init
ipv6_offload_init(void)
323 if (tcpv6_offload_init() < 0)
324 pr_crit("%s: Cannot add TCP protocol offload\n", __func__
);
325 if (udp_offload_init() < 0)
326 pr_crit("%s: Cannot add UDP protocol offload\n", __func__
);
327 if (ipv6_exthdrs_offload_init() < 0)
328 pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__
);
330 dev_add_offload(&ipv6_packet_offload
);
332 inet_add_offload(&sit_offload
, IPPROTO_IPV6
);
337 fs_initcall(ipv6_offload_init
);