2 * IPV4 GSO/GRO offload support
3 * Linux INET implementation
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version
8 * 2 of the License, or (at your option) any later version.
10 * TCPv4 GSO/GRO support
13 #include <linux/skbuff.h>
15 #include <net/protocol.h>
17 struct sk_buff
*tcp_gso_segment(struct sk_buff
*skb
,
18 netdev_features_t features
)
20 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
21 unsigned int sum_truesize
= 0;
28 struct sk_buff
*gso_skb
= skb
;
30 bool ooo_okay
, copy_destructor
;
32 if (!pskb_may_pull(skb
, sizeof(*th
)))
37 if (thlen
< sizeof(*th
))
40 if (!pskb_may_pull(skb
, thlen
))
43 oldlen
= (u16
)~skb
->len
;
44 __skb_pull(skb
, thlen
);
46 mss
= tcp_skb_mss(skb
);
47 if (unlikely(skb
->len
<= mss
))
50 if (skb_gso_ok(skb
, features
| NETIF_F_GSO_ROBUST
)) {
51 /* Packet is from an untrusted source, reset gso_segs. */
52 int type
= skb_shinfo(skb
)->gso_type
;
65 SKB_GSO_UDP_TUNNEL_CSUM
|
67 !(type
& (SKB_GSO_TCPV4
| SKB_GSO_TCPV6
))))
70 skb_shinfo(skb
)->gso_segs
= DIV_ROUND_UP(skb
->len
, mss
);
76 copy_destructor
= gso_skb
->destructor
== tcp_wfree
;
77 ooo_okay
= gso_skb
->ooo_okay
;
78 /* All segments but the first should have ooo_okay cleared */
81 segs
= skb_segment(skb
, features
);
85 /* Only first segment might have ooo_okay set */
86 segs
->ooo_okay
= ooo_okay
;
88 delta
= htonl(oldlen
+ (thlen
+ mss
));
94 newcheck
= ~csum_fold((__force __wsum
)((__force u32
)th
->check
+
98 th
->fin
= th
->psh
= 0;
101 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
102 th
->check
= gso_make_checksum(skb
, ~th
->check
);
105 if (copy_destructor
) {
106 skb
->destructor
= gso_skb
->destructor
;
107 skb
->sk
= gso_skb
->sk
;
108 sum_truesize
+= skb
->truesize
;
113 th
->seq
= htonl(seq
);
117 /* Following permits TCP Small Queues to work well with GSO :
118 * The callback to TCP stack will be called at the time last frag
119 * is freed at TX completion, and not right now when gso_skb
120 * is freed by GSO engine
122 if (copy_destructor
) {
123 swap(gso_skb
->sk
, skb
->sk
);
124 swap(gso_skb
->destructor
, skb
->destructor
);
125 sum_truesize
+= skb
->truesize
;
126 atomic_add(sum_truesize
- gso_skb
->truesize
,
127 &skb
->sk
->sk_wmem_alloc
);
130 delta
= htonl(oldlen
+ (skb_tail_pointer(skb
) -
131 skb_transport_header(skb
)) +
133 th
->check
= ~csum_fold((__force __wsum
)((__force u32
)th
->check
+
134 (__force u32
)delta
));
135 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
136 th
->check
= gso_make_checksum(skb
, ~th
->check
);
141 struct sk_buff
**tcp_gro_receive(struct sk_buff
**head
, struct sk_buff
*skb
)
143 struct sk_buff
**pp
= NULL
;
150 unsigned int mss
= 1;
156 off
= skb_gro_offset(skb
);
157 hlen
= off
+ sizeof(*th
);
158 th
= skb_gro_header_fast(skb
, off
);
159 if (skb_gro_header_hard(skb
, hlen
)) {
160 th
= skb_gro_header_slow(skb
, hlen
, off
);
165 thlen
= th
->doff
* 4;
166 if (thlen
< sizeof(*th
))
170 if (skb_gro_header_hard(skb
, hlen
)) {
171 th
= skb_gro_header_slow(skb
, hlen
, off
);
176 skb_gro_pull(skb
, thlen
);
178 len
= skb_gro_len(skb
);
179 flags
= tcp_flag_word(th
);
181 for (; (p
= *head
); head
= &p
->next
) {
182 if (!NAPI_GRO_CB(p
)->same_flow
)
187 if (*(u32
*)&th
->source
^ *(u32
*)&th2
->source
) {
188 NAPI_GRO_CB(p
)->same_flow
= 0;
195 goto out_check_final
;
198 /* Include the IP ID check below from the inner most IP hdr */
199 flush
= NAPI_GRO_CB(p
)->flush
| NAPI_GRO_CB(p
)->flush_id
;
200 flush
|= (__force
int)(flags
& TCP_FLAG_CWR
);
201 flush
|= (__force
int)((flags
^ tcp_flag_word(th2
)) &
202 ~(TCP_FLAG_CWR
| TCP_FLAG_FIN
| TCP_FLAG_PSH
));
203 flush
|= (__force
int)(th
->ack_seq
^ th2
->ack_seq
);
204 for (i
= sizeof(*th
); i
< thlen
; i
+= 4)
205 flush
|= *(u32
*)((u8
*)th
+ i
) ^
206 *(u32
*)((u8
*)th2
+ i
);
208 mss
= tcp_skb_mss(p
);
210 flush
|= (len
- 1) >= mss
;
211 flush
|= (ntohl(th2
->seq
) + skb_gro_len(p
)) ^ ntohl(th
->seq
);
213 if (flush
|| skb_gro_receive(head
, skb
)) {
215 goto out_check_final
;
220 tcp_flag_word(th2
) |= flags
& (TCP_FLAG_FIN
| TCP_FLAG_PSH
);
224 flush
|= (__force
int)(flags
& (TCP_FLAG_URG
| TCP_FLAG_PSH
|
225 TCP_FLAG_RST
| TCP_FLAG_SYN
|
228 if (p
&& (!NAPI_GRO_CB(skb
)->same_flow
|| flush
))
232 NAPI_GRO_CB(skb
)->flush
|= (flush
!= 0);
237 int tcp_gro_complete(struct sk_buff
*skb
)
239 struct tcphdr
*th
= tcp_hdr(skb
);
241 skb
->csum_start
= (unsigned char *)th
- skb
->head
;
242 skb
->csum_offset
= offsetof(struct tcphdr
, check
);
243 skb
->ip_summed
= CHECKSUM_PARTIAL
;
245 skb_shinfo(skb
)->gso_segs
= NAPI_GRO_CB(skb
)->count
;
248 skb_shinfo(skb
)->gso_type
|= SKB_GSO_TCP_ECN
;
252 EXPORT_SYMBOL(tcp_gro_complete
);
254 static int tcp_v4_gso_send_check(struct sk_buff
*skb
)
256 const struct iphdr
*iph
;
259 if (!pskb_may_pull(skb
, sizeof(*th
)))
266 skb
->ip_summed
= CHECKSUM_PARTIAL
;
267 __tcp_v4_send_check(skb
, iph
->saddr
, iph
->daddr
);
271 static struct sk_buff
**tcp4_gro_receive(struct sk_buff
**head
, struct sk_buff
*skb
)
273 /* Use the IP hdr immediately proceeding for this transport */
274 const struct iphdr
*iph
= skb_gro_network_header(skb
);
277 /* Don't bother verifying checksum if we're going to flush anyway. */
278 if (NAPI_GRO_CB(skb
)->flush
)
281 wsum
= NAPI_GRO_CB(skb
)->csum
;
283 switch (skb
->ip_summed
) {
285 wsum
= skb_checksum(skb
, skb_gro_offset(skb
), skb_gro_len(skb
),
290 case CHECKSUM_COMPLETE
:
291 if (!tcp_v4_check(skb_gro_len(skb
), iph
->saddr
, iph
->daddr
,
293 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
297 NAPI_GRO_CB(skb
)->flush
= 1;
302 return tcp_gro_receive(head
, skb
);
305 static int tcp4_gro_complete(struct sk_buff
*skb
, int thoff
)
307 const struct iphdr
*iph
= ip_hdr(skb
);
308 struct tcphdr
*th
= tcp_hdr(skb
);
310 th
->check
= ~tcp_v4_check(skb
->len
- thoff
, iph
->saddr
,
312 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
314 return tcp_gro_complete(skb
);
317 static const struct net_offload tcpv4_offload
= {
319 .gso_send_check
= tcp_v4_gso_send_check
,
320 .gso_segment
= tcp_gso_segment
,
321 .gro_receive
= tcp4_gro_receive
,
322 .gro_complete
= tcp4_gro_complete
,
326 int __init
tcpv4_offload_init(void)
328 return inet_add_offload(&tcpv4_offload
, IPPROTO_TCP
);