Commit | Line | Data |
---|---|---|
da5bab07 DB |
1 | /* |
2 | * IPV4 GSO/GRO offload support | |
3 | * Linux INET implementation | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or | |
6 | * modify it under the terms of the GNU General Public License | |
7 | * as published by the Free Software Foundation; either version | |
8 | * 2 of the License, or (at your option) any later version. | |
9 | * | |
10 | * UDPv4 GSO support | |
11 | */ | |
12 | ||
13 | #include <linux/skbuff.h> | |
14 | #include <net/udp.h> | |
15 | #include <net/protocol.h> | |
16 | ||
b582ef09 | 17 | static DEFINE_SPINLOCK(udp_offload_lock); |
a1d0cd8e | 18 | static struct udp_offload_priv __rcu *udp_offload_base __read_mostly; |
b582ef09 | 19 | |
a664a4f7 SP |
20 | #define udp_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&udp_offload_lock)) |
21 | ||
b582ef09 OG |
22 | struct udp_offload_priv { |
23 | struct udp_offload *offload; | |
24 | struct rcu_head rcu; | |
25 | struct udp_offload_priv __rcu *next; | |
26 | }; | |
27 | ||
8bce6d7d TH |
28 | static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb, |
29 | netdev_features_t features, | |
30 | struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, | |
31 | netdev_features_t features), | |
32 | __be16 new_protocol) | |
155e010e TH |
33 | { |
34 | struct sk_buff *segs = ERR_PTR(-EINVAL); | |
35 | u16 mac_offset = skb->mac_header; | |
36 | int mac_len = skb->mac_len; | |
37 | int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb); | |
38 | __be16 protocol = skb->protocol; | |
39 | netdev_features_t enc_features; | |
40 | int udp_offset, outer_hlen; | |
41 | unsigned int oldlen; | |
42 | bool need_csum; | |
43 | ||
44 | oldlen = (u16)~skb->len; | |
45 | ||
46 | if (unlikely(!pskb_may_pull(skb, tnl_hlen))) | |
47 | goto out; | |
48 | ||
49 | skb->encapsulation = 0; | |
50 | __skb_pull(skb, tnl_hlen); | |
51 | skb_reset_mac_header(skb); | |
52 | skb_set_network_header(skb, skb_inner_network_offset(skb)); | |
53 | skb->mac_len = skb_inner_network_offset(skb); | |
8bce6d7d | 54 | skb->protocol = new_protocol; |
155e010e TH |
55 | |
56 | need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM); | |
57 | if (need_csum) | |
58 | skb->encap_hdr_csum = 1; | |
59 | ||
60 | /* segment inner packet. */ | |
1e16aa3d | 61 | enc_features = skb->dev->hw_enc_features & features; |
8bce6d7d | 62 | segs = gso_inner_segment(skb, enc_features); |
27446442 | 63 | if (IS_ERR_OR_NULL(segs)) { |
155e010e TH |
64 | skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset, |
65 | mac_len); | |
66 | goto out; | |
67 | } | |
68 | ||
69 | outer_hlen = skb_tnl_header_len(skb); | |
70 | udp_offset = outer_hlen - tnl_hlen; | |
71 | skb = segs; | |
72 | do { | |
73 | struct udphdr *uh; | |
74 | int len; | |
75 | ||
76 | skb_reset_inner_headers(skb); | |
77 | skb->encapsulation = 1; | |
78 | ||
79 | skb->mac_len = mac_len; | |
80 | ||
81 | skb_push(skb, outer_hlen); | |
82 | skb_reset_mac_header(skb); | |
83 | skb_set_network_header(skb, mac_len); | |
84 | skb_set_transport_header(skb, udp_offset); | |
85 | len = skb->len - udp_offset; | |
86 | uh = udp_hdr(skb); | |
87 | uh->len = htons(len); | |
88 | ||
89 | if (need_csum) { | |
90 | __be32 delta = htonl(oldlen + len); | |
91 | ||
92 | uh->check = ~csum_fold((__force __wsum) | |
93 | ((__force u32)uh->check + | |
94 | (__force u32)delta)); | |
95 | uh->check = gso_make_checksum(skb, ~uh->check); | |
96 | ||
97 | if (uh->check == 0) | |
98 | uh->check = CSUM_MANGLED_0; | |
99 | } | |
100 | ||
101 | skb->protocol = protocol; | |
102 | } while ((skb = skb->next)); | |
103 | out: | |
104 | return segs; | |
105 | } | |
106 | ||
8bce6d7d TH |
107 | struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, |
108 | netdev_features_t features, | |
109 | bool is_ipv6) | |
110 | { | |
111 | __be16 protocol = skb->protocol; | |
112 | const struct net_offload **offloads; | |
113 | const struct net_offload *ops; | |
114 | struct sk_buff *segs = ERR_PTR(-EINVAL); | |
115 | struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, | |
116 | netdev_features_t features); | |
117 | ||
118 | rcu_read_lock(); | |
119 | ||
120 | switch (skb->inner_protocol_type) { | |
121 | case ENCAP_TYPE_ETHER: | |
122 | protocol = skb->inner_protocol; | |
123 | gso_inner_segment = skb_mac_gso_segment; | |
124 | break; | |
125 | case ENCAP_TYPE_IPPROTO: | |
126 | offloads = is_ipv6 ? inet6_offloads : inet_offloads; | |
127 | ops = rcu_dereference(offloads[skb->inner_ipproto]); | |
128 | if (!ops || !ops->callbacks.gso_segment) | |
129 | goto out_unlock; | |
130 | gso_inner_segment = ops->callbacks.gso_segment; | |
131 | break; | |
132 | default: | |
133 | goto out_unlock; | |
134 | } | |
135 | ||
136 | segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment, | |
137 | protocol); | |
138 | ||
139 | out_unlock: | |
140 | rcu_read_unlock(); | |
141 | ||
142 | return segs; | |
143 | } | |
144 | ||
da5bab07 DB |
145 | static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, |
146 | netdev_features_t features) | |
147 | { | |
148 | struct sk_buff *segs = ERR_PTR(-EINVAL); | |
149 | unsigned int mss; | |
7a7ffbab | 150 | __wsum csum; |
f71470b3 TH |
151 | struct udphdr *uh; |
152 | struct iphdr *iph; | |
7a7ffbab WCC |
153 | |
154 | if (skb->encapsulation && | |
0f4f4ffa TH |
155 | (skb_shinfo(skb)->gso_type & |
156 | (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) { | |
8bce6d7d | 157 | segs = skb_udp_tunnel_segment(skb, features, false); |
7a7ffbab WCC |
158 | goto out; |
159 | } | |
da5bab07 | 160 | |
f71470b3 TH |
161 | if (!pskb_may_pull(skb, sizeof(struct udphdr))) |
162 | goto out; | |
163 | ||
da5bab07 DB |
164 | mss = skb_shinfo(skb)->gso_size; |
165 | if (unlikely(skb->len <= mss)) | |
166 | goto out; | |
167 | ||
168 | if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) { | |
169 | /* Packet is from an untrusted source, reset gso_segs. */ | |
170 | int type = skb_shinfo(skb)->gso_type; | |
171 | ||
172 | if (unlikely(type & ~(SKB_GSO_UDP | SKB_GSO_DODGY | | |
173 | SKB_GSO_UDP_TUNNEL | | |
0f4f4ffa | 174 | SKB_GSO_UDP_TUNNEL_CSUM | |
cb32f511 | 175 | SKB_GSO_IPIP | |
4749c09c TH |
176 | SKB_GSO_GRE | SKB_GSO_GRE_CSUM | |
177 | SKB_GSO_MPLS) || | |
da5bab07 DB |
178 | !(type & (SKB_GSO_UDP)))) |
179 | goto out; | |
180 | ||
181 | skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss); | |
182 | ||
183 | segs = NULL; | |
184 | goto out; | |
185 | } | |
186 | ||
7a7ffbab WCC |
187 | /* Do software UFO. Complete and fill in the UDP checksum as |
188 | * HW cannot do checksum of UDP packets sent as multiple | |
189 | * IP fragments. | |
190 | */ | |
f71470b3 TH |
191 | |
192 | uh = udp_hdr(skb); | |
193 | iph = ip_hdr(skb); | |
194 | ||
195 | uh->check = 0; | |
196 | csum = skb_checksum(skb, 0, skb->len, 0); | |
197 | uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum); | |
198 | if (uh->check == 0) | |
199 | uh->check = CSUM_MANGLED_0; | |
200 | ||
7a7ffbab WCC |
201 | skb->ip_summed = CHECKSUM_NONE; |
202 | ||
da5bab07 DB |
203 | /* Fragment the skb. IP headers of the fragments are updated in |
204 | * inet_gso_segment() | |
205 | */ | |
7a7ffbab | 206 | segs = skb_segment(skb, features); |
da5bab07 DB |
207 | out: |
208 | return segs; | |
209 | } | |
210 | ||
b582ef09 OG |
211 | int udp_add_offload(struct udp_offload *uo) |
212 | { | |
b5aaab12 | 213 | struct udp_offload_priv *new_offload = kzalloc(sizeof(*new_offload), GFP_ATOMIC); |
b582ef09 OG |
214 | |
215 | if (!new_offload) | |
216 | return -ENOMEM; | |
217 | ||
218 | new_offload->offload = uo; | |
219 | ||
220 | spin_lock(&udp_offload_lock); | |
a664a4f7 SP |
221 | new_offload->next = udp_offload_base; |
222 | rcu_assign_pointer(udp_offload_base, new_offload); | |
b582ef09 OG |
223 | spin_unlock(&udp_offload_lock); |
224 | ||
225 | return 0; | |
226 | } | |
227 | EXPORT_SYMBOL(udp_add_offload); | |
228 | ||
229 | static void udp_offload_free_routine(struct rcu_head *head) | |
230 | { | |
231 | struct udp_offload_priv *ou_priv = container_of(head, struct udp_offload_priv, rcu); | |
232 | kfree(ou_priv); | |
233 | } | |
234 | ||
235 | void udp_del_offload(struct udp_offload *uo) | |
236 | { | |
237 | struct udp_offload_priv __rcu **head = &udp_offload_base; | |
238 | struct udp_offload_priv *uo_priv; | |
239 | ||
240 | spin_lock(&udp_offload_lock); | |
241 | ||
a664a4f7 | 242 | uo_priv = udp_deref_protected(*head); |
b582ef09 | 243 | for (; uo_priv != NULL; |
a664a4f7 | 244 | uo_priv = udp_deref_protected(*head)) { |
b582ef09 | 245 | if (uo_priv->offload == uo) { |
a664a4f7 SP |
246 | rcu_assign_pointer(*head, |
247 | udp_deref_protected(uo_priv->next)); | |
b582ef09 OG |
248 | goto unlock; |
249 | } | |
250 | head = &uo_priv->next; | |
251 | } | |
a1d0cd8e | 252 | pr_warn("udp_del_offload: didn't find offload for port %d\n", ntohs(uo->port)); |
b582ef09 OG |
253 | unlock: |
254 | spin_unlock(&udp_offload_lock); | |
255 | if (uo_priv != NULL) | |
256 | call_rcu(&uo_priv->rcu, udp_offload_free_routine); | |
257 | } | |
258 | EXPORT_SYMBOL(udp_del_offload); | |
259 | ||
57c67ff4 TH |
260 | struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, |
261 | struct udphdr *uh) | |
b582ef09 OG |
262 | { |
263 | struct udp_offload_priv *uo_priv; | |
264 | struct sk_buff *p, **pp = NULL; | |
57c67ff4 TH |
265 | struct udphdr *uh2; |
266 | unsigned int off = skb_gro_offset(skb); | |
b582ef09 OG |
267 | int flush = 1; |
268 | ||
269 | if (NAPI_GRO_CB(skb)->udp_mark || | |
662880f4 TH |
270 | (skb->ip_summed != CHECKSUM_PARTIAL && |
271 | NAPI_GRO_CB(skb)->csum_cnt == 0 && | |
272 | !NAPI_GRO_CB(skb)->csum_valid)) | |
b582ef09 OG |
273 | goto out; |
274 | ||
275 | /* mark that this skb passed once through the udp gro layer */ | |
276 | NAPI_GRO_CB(skb)->udp_mark = 1; | |
b582ef09 OG |
277 | |
278 | rcu_read_lock(); | |
279 | uo_priv = rcu_dereference(udp_offload_base); | |
280 | for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { | |
281 | if (uo_priv->offload->port == uh->dest && | |
282 | uo_priv->offload->callbacks.gro_receive) | |
283 | goto unflush; | |
284 | } | |
285 | goto out_unlock; | |
286 | ||
287 | unflush: | |
288 | flush = 0; | |
289 | ||
290 | for (p = *head; p; p = p->next) { | |
291 | if (!NAPI_GRO_CB(p)->same_flow) | |
292 | continue; | |
293 | ||
294 | uh2 = (struct udphdr *)(p->data + off); | |
57c67ff4 TH |
295 | |
296 | /* Match ports and either checksums are either both zero | |
297 | * or nonzero. | |
298 | */ | |
299 | if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) || | |
300 | (!uh->check ^ !uh2->check)) { | |
b582ef09 OG |
301 | NAPI_GRO_CB(p)->same_flow = 0; |
302 | continue; | |
303 | } | |
304 | } | |
305 | ||
306 | skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */ | |
6bae1d4c | 307 | skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr)); |
afe93325 | 308 | NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; |
b582ef09 OG |
309 | pp = uo_priv->offload->callbacks.gro_receive(head, skb); |
310 | ||
311 | out_unlock: | |
312 | rcu_read_unlock(); | |
313 | out: | |
314 | NAPI_GRO_CB(skb)->flush |= flush; | |
315 | return pp; | |
316 | } | |
317 | ||
57c67ff4 TH |
318 | static struct sk_buff **udp4_gro_receive(struct sk_buff **head, |
319 | struct sk_buff *skb) | |
320 | { | |
321 | struct udphdr *uh = udp_gro_udphdr(skb); | |
322 | ||
2abb7cdc TH |
323 | if (unlikely(!uh)) |
324 | goto flush; | |
57c67ff4 | 325 | |
2abb7cdc | 326 | /* Don't bother verifying checksum if we're going to flush anyway. */ |
2d8f7e2c | 327 | if (NAPI_GRO_CB(skb)->flush) |
2abb7cdc TH |
328 | goto skip; |
329 | ||
330 | if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check, | |
331 | inet_gro_compute_pseudo)) | |
332 | goto flush; | |
333 | else if (uh->check) | |
334 | skb_gro_checksum_try_convert(skb, IPPROTO_UDP, uh->check, | |
335 | inet_gro_compute_pseudo); | |
336 | skip: | |
efc98d08 | 337 | NAPI_GRO_CB(skb)->is_ipv6 = 0; |
57c67ff4 | 338 | return udp_gro_receive(head, skb, uh); |
2abb7cdc TH |
339 | |
340 | flush: | |
341 | NAPI_GRO_CB(skb)->flush = 1; | |
342 | return NULL; | |
57c67ff4 TH |
343 | } |
344 | ||
345 | int udp_gro_complete(struct sk_buff *skb, int nhoff) | |
b582ef09 OG |
346 | { |
347 | struct udp_offload_priv *uo_priv; | |
348 | __be16 newlen = htons(skb->len - nhoff); | |
349 | struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); | |
350 | int err = -ENOSYS; | |
351 | ||
352 | uh->len = newlen; | |
353 | ||
354 | rcu_read_lock(); | |
355 | ||
356 | uo_priv = rcu_dereference(udp_offload_base); | |
357 | for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { | |
358 | if (uo_priv->offload->port == uh->dest && | |
359 | uo_priv->offload->callbacks.gro_complete) | |
360 | break; | |
361 | } | |
362 | ||
afe93325 TH |
363 | if (uo_priv != NULL) { |
364 | NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; | |
b582ef09 | 365 | err = uo_priv->offload->callbacks.gro_complete(skb, nhoff + sizeof(struct udphdr)); |
afe93325 | 366 | } |
b582ef09 OG |
367 | |
368 | rcu_read_unlock(); | |
369 | return err; | |
370 | } | |
371 | ||
72bb17b3 | 372 | static int udp4_gro_complete(struct sk_buff *skb, int nhoff) |
57c67ff4 TH |
373 | { |
374 | const struct iphdr *iph = ip_hdr(skb); | |
375 | struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); | |
376 | ||
377 | if (uh->check) | |
378 | uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr, | |
379 | iph->daddr, 0); | |
380 | ||
381 | return udp_gro_complete(skb, nhoff); | |
382 | } | |
383 | ||
da5bab07 DB |
384 | static const struct net_offload udpv4_offload = { |
385 | .callbacks = { | |
da5bab07 | 386 | .gso_segment = udp4_ufo_fragment, |
57c67ff4 TH |
387 | .gro_receive = udp4_gro_receive, |
388 | .gro_complete = udp4_gro_complete, | |
da5bab07 DB |
389 | }, |
390 | }; | |
391 | ||
392 | int __init udpv4_offload_init(void) | |
393 | { | |
394 | return inet_add_offload(&udpv4_offload, IPPROTO_UDP); | |
395 | } |