projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
packet: minor: add generic tpacket_uhdr to access packet headers
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
1da177e4
LT		 91
92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
ce06b03e
DM		 240/* register_prot_hook must be invoked with the po->bind_lock held,
241 * or from a context in which asynchronous accesses to the packet
242 * socket is not possible (packet_create()).
243 */
244static void register_prot_hook(struct sock *sk)
245{
246 struct packet_sock *po = pkt_sk(sk);
247 if (!po->running) {
dc99f600
DM		 248 if (po->fanout)
249 __fanout_link(sk, po);
250 else
251 dev_add_pack(&po->prot_hook);
ce06b03e
DM		 252 sock_hold(sk);
253 po->running = 1;
254 }
255}
256
257/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
258 * held. If the sync parameter is true, we will temporarily drop
259 * the po->bind_lock and do a synchronize_net to make sure no
260 * asynchronous packet processing paths still refer to the elements
261 * of po->prot_hook. If the sync parameter is false, it is the
262 * callers responsibility to take care of this.
263 */
264static void __unregister_prot_hook(struct sock *sk, bool sync)
265{
266 struct packet_sock *po = pkt_sk(sk);
267
268 po->running = 0;
dc99f600
DM		 269 if (po->fanout)
270 __fanout_unlink(sk, po);
271 else
272 __dev_remove_pack(&po->prot_hook);
ce06b03e
DM		 273 __sock_put(sk);
274
275 if (sync) {
276 spin_unlock(&po->bind_lock);
277 synchronize_net();
278 spin_lock(&po->bind_lock);
279 }
280}
281
282static void unregister_prot_hook(struct sock *sk, bool sync)
283{
284 struct packet_sock *po = pkt_sk(sk);
285
286 if (po->running)
287 __unregister_prot_hook(sk, sync);
288}
289
f6dafa95 290static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 291{
292 if (is_vmalloc_addr(addr))
293 return vmalloc_to_page(addr);
294 return virt_to_page(addr);
295}
296
69e3c75f 297static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 298{
184f489e 299 union tpacket_uhdr h;
1da177e4 300
69e3c75f 301 h.raw = frame;
bbd6ef87
PM		 302 switch (po->tp_version) {
303 case TPACKET_V1:
69e3c75f 304 h.h1->tp_status = status;
0af55bb5 305 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 306 break;
307 case TPACKET_V2:
69e3c75f 308 h.h2->tp_status = status;
0af55bb5 309 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 310 break;
f6fb8f10 311 case TPACKET_V3:
69e3c75f 312 default:
f6fb8f10 313 WARN(1, "TPACKET version not supported.\n");
69e3c75f 314 BUG();
bbd6ef87 315 }
69e3c75f
JB		 316
317 smp_wmb();
bbd6ef87
PM		 318}
319
69e3c75f 320static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 321{
184f489e 322 union tpacket_uhdr h;
bbd6ef87 323
69e3c75f
JB		 324 smp_rmb();
325
bbd6ef87
PM		 326 h.raw = frame;
327 switch (po->tp_version) {
328 case TPACKET_V1:
0af55bb5 329 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 330 return h.h1->tp_status;
bbd6ef87 331 case TPACKET_V2:
0af55bb5 332 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 333 return h.h2->tp_status;
f6fb8f10 334 case TPACKET_V3:
69e3c75f 335 default:
f6fb8f10 336 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 337 BUG();
338 return 0;
bbd6ef87 339 }
1da177e4 340}
69e3c75f
JB		 341
342static void *packet_lookup_frame(struct packet_sock *po,
343 struct packet_ring_buffer *rb,
344 unsigned int position,
345 int status)
346{
347 unsigned int pg_vec_pos, frame_offset;
184f489e 348 union tpacket_uhdr h;
69e3c75f
JB		 349
350 pg_vec_pos = position / rb->frames_per_block;
351 frame_offset = position % rb->frames_per_block;
352
0e3125c7
NH		 353 h.raw = rb->pg_vec[pg_vec_pos].buffer +
354 (frame_offset * rb->frame_size);
69e3c75f
JB		 355
356 if (status != __packet_get_status(po, h.raw))
357 return NULL;
358
359 return h.raw;
360}
361
eea49cc9 362static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 363 struct packet_ring_buffer *rb,
364 int status)
365{
366 return packet_lookup_frame(po, rb, rb->head, status);
367}
368
bc59ba39 369static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 370{
371 del_timer_sync(&pkc->retire_blk_timer);
372}
373
374static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
375 int tx_ring,
376 struct sk_buff_head *rb_queue)
377{
bc59ba39 378 struct tpacket_kbdq_core *pkc;
f6fb8f10 379
380 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
381
382 spin_lock(&rb_queue->lock);
383 pkc->delete_blk_timer = 1;
384 spin_unlock(&rb_queue->lock);
385
386 prb_del_retire_blk_timer(pkc);
387}
388
389static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 390 struct tpacket_kbdq_core *pkc,
f6fb8f10 391 void (*func) (unsigned long))
392{
393 init_timer(&pkc->retire_blk_timer);
394 pkc->retire_blk_timer.data = (long)po;
395 pkc->retire_blk_timer.function = func;
396 pkc->retire_blk_timer.expires = jiffies;
397}
398
399static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
400{
bc59ba39 401 struct tpacket_kbdq_core *pkc;
f6fb8f10 402
403 if (tx_ring)
404 BUG();
405
406 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
407 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
408}
409
410static int prb_calc_retire_blk_tmo(struct packet_sock *po,
411 int blk_size_in_bytes)
412{
413 struct net_device *dev;
414 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 415 struct ethtool_cmd ecmd;
416 int err;
e440cf2c 417 u32 speed;
f6fb8f10 418
4bc71cb9
JP		 419 rtnl_lock();
420 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
421 if (unlikely(!dev)) {
422 rtnl_unlock();
f6fb8f10 423 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 424 }
425 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 426 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 427 rtnl_unlock();
428 if (!err) {
4bc71cb9
JP		 429 /*
430 * If the link speed is so slow you don't really
431 * need to worry about perf anyways
432 */
e440cf2c 433 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 434 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 435 } else {
436 msec = 1;
437 div = speed / 1000;
f6fb8f10 438 }
439 }
440
441 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
442
443 if (div)
444 mbits /= div;
445
446 tmo = mbits * msec;
447
448 if (div)
449 return tmo+1;
450 return tmo;
451}
452
bc59ba39 453static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 454 union tpacket_req_u *req_u)
455{
456 p1->feature_req_word = req_u->req3.tp_feature_req_word;
457}
458
459static void init_prb_bdqc(struct packet_sock *po,
460 struct packet_ring_buffer *rb,
461 struct pgv *pg_vec,
462 union tpacket_req_u *req_u, int tx_ring)
463{
bc59ba39 464 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
465 struct tpacket_block_desc *pbd;
f6fb8f10 466
467 memset(p1, 0x0, sizeof(*p1));
468
469 p1->knxt_seq_num = 1;
470 p1->pkbdq = pg_vec;
bc59ba39 471 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 472 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 473 p1->kblk_size = req_u->req3.tp_block_size;
474 p1->knum_blocks = req_u->req3.tp_block_nr;
475 p1->hdrlen = po->tp_hdrlen;
476 p1->version = po->tp_version;
477 p1->last_kactive_blk_num = 0;
478 po->stats_u.stats3.tp_freeze_q_cnt = 0;
479 if (req_u->req3.tp_retire_blk_tov)
480 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
481 else
482 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
483 req_u->req3.tp_block_size);
484 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
485 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
486
487 prb_init_ft_ops(p1, req_u);
488 prb_setup_retire_blk_timer(po, tx_ring);
489 prb_open_block(p1, pbd);
490}
491
492/* Do NOT update the last_blk_num first.
493 * Assumes sk_buff_head lock is held.
494 */
bc59ba39 495static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 496{
497 mod_timer(&pkc->retire_blk_timer,
498 jiffies + pkc->tov_in_jiffies);
499 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
500}
501
502/*
503 * Timer logic:
504 * 1) We refresh the timer only when we open a block.
505 * By doing this we don't waste cycles refreshing the timer
506 * on packet-by-packet basis.
507 *
508 * With a 1MB block-size, on a 1Gbps line, it will take
509 * i) ~8 ms to fill a block + ii) memcpy etc.
510 * In this cut we are not accounting for the memcpy time.
511 *
512 * So, if the user sets the 'tmo' to 10ms then the timer
513 * will never fire while the block is still getting filled
514 * (which is what we want). However, the user could choose
515 * to close a block early and that's fine.
516 *
517 * But when the timer does fire, we check whether or not to refresh it.
518 * Since the tmo granularity is in msecs, it is not too expensive
519 * to refresh the timer, lets say every '8' msecs.
520 * Either the user can set the 'tmo' or we can derive it based on
521 * a) line-speed and b) block-size.
522 * prb_calc_retire_blk_tmo() calculates the tmo.
523 *
524 */
525static void prb_retire_rx_blk_timer_expired(unsigned long data)
526{
527 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 528 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 529 unsigned int frozen;
bc59ba39 530 struct tpacket_block_desc *pbd;
f6fb8f10 531
532 spin_lock(&po->sk.sk_receive_queue.lock);
533
534 frozen = prb_queue_frozen(pkc);
535 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
536
537 if (unlikely(pkc->delete_blk_timer))
538 goto out;
539
540 /* We only need to plug the race when the block is partially filled.
541 * tpacket_rcv:
542 * lock(); increment BLOCK_NUM_PKTS; unlock()
543 * copy_bits() is in progress ...
544 * timer fires on other cpu:
545 * we can't retire the current block because copy_bits
546 * is in progress.
547 *
548 */
549 if (BLOCK_NUM_PKTS(pbd)) {
550 while (atomic_read(&pkc->blk_fill_in_prog)) {
551 /* Waiting for skb_copy_bits to finish... */
552 cpu_relax();
553 }
554 }
555
556 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
557 if (!frozen) {
558 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
559 if (!prb_dispatch_next_block(pkc, po))
560 goto refresh_timer;
561 else
562 goto out;
563 } else {
564 /* Case 1. Queue was frozen because user-space was
565 * lagging behind.
566 */
567 if (prb_curr_blk_in_use(pkc, pbd)) {
568 /*
569 * Ok, user-space is still behind.
570 * So just refresh the timer.
571 */
572 goto refresh_timer;
573 } else {
574 /* Case 2. queue was frozen,user-space caught up,
575 * now the link went idle && the timer fired.
576 * We don't have a block to close.So we open this
577 * block and restart the timer.
578 * opening a block thaws the queue,restarts timer
579 * Thawing/timer-refresh is a side effect.
580 */
581 prb_open_block(pkc, pbd);
582 goto out;
583 }
584 }
585 }
586
587refresh_timer:
588 _prb_refresh_rx_retire_blk_timer(pkc);
589
590out:
591 spin_unlock(&po->sk.sk_receive_queue.lock);
592}
593
eea49cc9 594static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 595 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 596{
597 /* Flush everything minus the block header */
598
599#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
600 u8 *start, *end;
601
602 start = (u8 *)pbd1;
603
604 /* Skip the block header(we know header WILL fit in 4K) */
605 start += PAGE_SIZE;
606
607 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
608 for (; start < end; start += PAGE_SIZE)
609 flush_dcache_page(pgv_to_page(start));
610
611 smp_wmb();
612#endif
613
614 /* Now update the block status. */
615
616 BLOCK_STATUS(pbd1) = status;
617
618 /* Flush the block header */
619
620#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
621 start = (u8 *)pbd1;
622 flush_dcache_page(pgv_to_page(start));
623
624 smp_wmb();
625#endif
626}
627
628/*
629 * Side effect:
630 *
631 * 1) flush the block
632 * 2) Increment active_blk_num
633 *
634 * Note:We DONT refresh the timer on purpose.
635 * Because almost always the next block will be opened.
636 */
bc59ba39 637static void prb_close_block(struct tpacket_kbdq_core *pkc1,
638 struct tpacket_block_desc *pbd1,
f6fb8f10 639 struct packet_sock *po, unsigned int stat)
640{
641 __u32 status = TP_STATUS_USER | stat;
642
643 struct tpacket3_hdr *last_pkt;
bc59ba39 644 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 645
646 if (po->stats.tp_drops)
647 status |= TP_STATUS_LOSING;
648
649 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
650 last_pkt->tp_next_offset = 0;
651
652 /* Get the ts of the last pkt */
653 if (BLOCK_NUM_PKTS(pbd1)) {
654 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
655 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
656 } else {
657 /* Ok, we tmo'd - so get the current time */
658 struct timespec ts;
659 getnstimeofday(&ts);
660 h1->ts_last_pkt.ts_sec = ts.tv_sec;
661 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
662 }
663
664 smp_wmb();
665
666 /* Flush the block */
667 prb_flush_block(pkc1, pbd1, status);
668
669 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
670}
671
eea49cc9 672static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 673{
674 pkc->reset_pending_on_curr_blk = 0;
675}
676
677/*
678 * Side effect of opening a block:
679 *
680 * 1) prb_queue is thawed.
681 * 2) retire_blk_timer is refreshed.
682 *
683 */
bc59ba39 684static void prb_open_block(struct tpacket_kbdq_core *pkc1,
685 struct tpacket_block_desc *pbd1)
f6fb8f10 686{
687 struct timespec ts;
bc59ba39 688 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 689
690 smp_rmb();
691
692 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd1))) {
693
694 /* We could have just memset this but we will lose the
695 * flexibility of making the priv area sticky
696 */
697 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
698 BLOCK_NUM_PKTS(pbd1) = 0;
699 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
700 getnstimeofday(&ts);
701 h1->ts_first_pkt.ts_sec = ts.tv_sec;
702 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
703 pkc1->pkblk_start = (char *)pbd1;
e3192690 704 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 705 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
706 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
707 pbd1->version = pkc1->version;
708 pkc1->prev = pkc1->nxt_offset;
709 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
710 prb_thaw_queue(pkc1);
711 _prb_refresh_rx_retire_blk_timer(pkc1);
712
713 smp_wmb();
714
715 return;
716 }
717
718 WARN(1, "ERROR block:%p is NOT FREE status:%d kactive_blk_num:%d\n",
719 pbd1, BLOCK_STATUS(pbd1), pkc1->kactive_blk_num);
720 dump_stack();
721 BUG();
722}
723
724/*
725 * Queue freeze logic:
726 * 1) Assume tp_block_nr = 8 blocks.
727 * 2) At time 't0', user opens Rx ring.
728 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
729 * 4) user-space is either sleeping or processing block '0'.
730 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
731 * it will close block-7,loop around and try to fill block '0'.
732 * call-flow:
733 * __packet_lookup_frame_in_block
734 * prb_retire_current_block()
735 * prb_dispatch_next_block()
736 * |->(BLOCK_STATUS == USER) evaluates to true
737 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
738 * 6) Now there are two cases:
739 * 6.1) Link goes idle right after the queue is frozen.
740 * But remember, the last open_block() refreshed the timer.
741 * When this timer expires,it will refresh itself so that we can
742 * re-open block-0 in near future.
743 * 6.2) Link is busy and keeps on receiving packets. This is a simple
744 * case and __packet_lookup_frame_in_block will check if block-0
745 * is free and can now be re-used.
746 */
eea49cc9 747static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 748 struct packet_sock *po)
749{
750 pkc->reset_pending_on_curr_blk = 1;
751 po->stats_u.stats3.tp_freeze_q_cnt++;
752}
753
754#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
755
756/*
757 * If the next block is free then we will dispatch it
758 * and return a good offset.
759 * Else, we will freeze the queue.
760 * So, caller must check the return value.
761 */
bc59ba39 762static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 763 struct packet_sock *po)
764{
bc59ba39 765 struct tpacket_block_desc *pbd;
f6fb8f10 766
767 smp_rmb();
768
769 /* 1. Get current block num */
770 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
771
772 /* 2. If this block is currently in_use then freeze the queue */
773 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
774 prb_freeze_queue(pkc, po);
775 return NULL;
776 }
777
778 /*
779 * 3.
780 * open this block and return the offset where the first packet
781 * needs to get stored.
782 */
783 prb_open_block(pkc, pbd);
784 return (void *)pkc->nxt_offset;
785}
786
bc59ba39 787static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 788 struct packet_sock *po, unsigned int status)
789{
bc59ba39 790 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 791
792 /* retire/close the current block */
793 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
794 /*
795 * Plug the case where copy_bits() is in progress on
796 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
797 * have space to copy the pkt in the current block and
798 * called prb_retire_current_block()
799 *
800 * We don't need to worry about the TMO case because
801 * the timer-handler already handled this case.
802 */
803 if (!(status & TP_STATUS_BLK_TMO)) {
804 while (atomic_read(&pkc->blk_fill_in_prog)) {
805 /* Waiting for skb_copy_bits to finish... */
806 cpu_relax();
807 }
808 }
809 prb_close_block(pkc, pbd, po, status);
810 return;
811 }
812
813 WARN(1, "ERROR-pbd[%d]:%p\n", pkc->kactive_blk_num, pbd);
814 dump_stack();
815 BUG();
816}
817
eea49cc9 818static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 819 struct tpacket_block_desc *pbd)
f6fb8f10 820{
821 return TP_STATUS_USER & BLOCK_STATUS(pbd);
822}
823
eea49cc9 824static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 825{
826 return pkc->reset_pending_on_curr_blk;
827}
828
eea49cc9 829static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 830{
bc59ba39 831 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 832 atomic_dec(&pkc->blk_fill_in_prog);
833}
834
eea49cc9 835static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 836 struct tpacket3_hdr *ppd)
837{
838 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
839}
840
eea49cc9 841static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 842 struct tpacket3_hdr *ppd)
843{
844 ppd->hv1.tp_rxhash = 0;
845}
846
eea49cc9 847static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 848 struct tpacket3_hdr *ppd)
849{
850 if (vlan_tx_tag_present(pkc->skb)) {
851 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
852 ppd->tp_status = TP_STATUS_VLAN_VALID;
853 } else {
9e67030a 854 ppd->hv1.tp_vlan_tci = 0;
855 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 856 }
857}
858
bc59ba39 859static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 860 struct tpacket3_hdr *ppd)
861{
862 prb_fill_vlan_info(pkc, ppd);
863
864 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
865 prb_fill_rxhash(pkc, ppd);
866 else
867 prb_clear_rxhash(pkc, ppd);
868}
869
eea49cc9 870static void prb_fill_curr_block(char *curr,
bc59ba39 871 struct tpacket_kbdq_core *pkc,
872 struct tpacket_block_desc *pbd,
f6fb8f10 873 unsigned int len)
874{
875 struct tpacket3_hdr *ppd;
876
877 ppd = (struct tpacket3_hdr *)curr;
878 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
879 pkc->prev = curr;
880 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
881 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
882 BLOCK_NUM_PKTS(pbd) += 1;
883 atomic_inc(&pkc->blk_fill_in_prog);
884 prb_run_all_ft_ops(pkc, ppd);
885}
886
887/* Assumes caller has the sk->rx_queue.lock */
888static void *__packet_lookup_frame_in_block(struct packet_sock *po,
889 struct sk_buff *skb,
890 int status,
891 unsigned int len
892 )
893{
bc59ba39 894 struct tpacket_kbdq_core *pkc;
895 struct tpacket_block_desc *pbd;
f6fb8f10 896 char *curr, *end;
897
e3192690 898 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 899 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
900
901 /* Queue is frozen when user space is lagging behind */
902 if (prb_queue_frozen(pkc)) {
903 /*
904 * Check if that last block which caused the queue to freeze,
905 * is still in_use by user-space.
906 */
907 if (prb_curr_blk_in_use(pkc, pbd)) {
908 /* Can't record this packet */
909 return NULL;
910 } else {
911 /*
912 * Ok, the block was released by user-space.
913 * Now let's open that block.
914 * opening a block also thaws the queue.
915 * Thawing is a side effect.
916 */
917 prb_open_block(pkc, pbd);
918 }
919 }
920
921 smp_mb();
922 curr = pkc->nxt_offset;
923 pkc->skb = skb;
e3192690 924 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 925
926 /* first try the current block */
927 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
928 prb_fill_curr_block(curr, pkc, pbd, len);
929 return (void *)curr;
930 }
931
932 /* Ok, close the current block */
933 prb_retire_current_block(pkc, po, 0);
934
935 /* Now, try to dispatch the next block */
936 curr = (char *)prb_dispatch_next_block(pkc, po);
937 if (curr) {
938 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
939 prb_fill_curr_block(curr, pkc, pbd, len);
940 return (void *)curr;
941 }
942
943 /*
944 * No free blocks are available.user_space hasn't caught up yet.
945 * Queue was just frozen and now this packet will get dropped.
946 */
947 return NULL;
948}
949
eea49cc9 950static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 951 struct sk_buff *skb,
952 int status, unsigned int len)
953{
954 char *curr = NULL;
955 switch (po->tp_version) {
956 case TPACKET_V1:
957 case TPACKET_V2:
958 curr = packet_lookup_frame(po, &po->rx_ring,
959 po->rx_ring.head, status);
960 return curr;
961 case TPACKET_V3:
962 return __packet_lookup_frame_in_block(po, skb, status, len);
963 default:
964 WARN(1, "TPACKET version not supported\n");
965 BUG();
99aa3473 966 return NULL;
f6fb8f10 967 }
968}
969
eea49cc9 970static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 971 struct packet_ring_buffer *rb,
77f65ebd 972 unsigned int idx,
f6fb8f10 973 int status)
974{
bc59ba39 975 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 976 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 977
978 if (status != BLOCK_STATUS(pbd))
979 return NULL;
980 return pbd;
981}
982
eea49cc9 983static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 984{
985 unsigned int prev;
986 if (rb->prb_bdqc.kactive_blk_num)
987 prev = rb->prb_bdqc.kactive_blk_num-1;
988 else
989 prev = rb->prb_bdqc.knum_blocks-1;
990 return prev;
991}
992
993/* Assumes caller has held the rx_queue.lock */
eea49cc9 994static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 995 struct packet_ring_buffer *rb,
996 int status)
997{
998 unsigned int previous = prb_previous_blk_num(rb);
999 return prb_lookup_block(po, rb, previous, status);
1000}
1001
eea49cc9 1002static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1003 struct packet_ring_buffer *rb,
1004 int status)
1005{
1006 if (po->tp_version <= TPACKET_V2)
1007 return packet_previous_frame(po, rb, status);
1008
1009 return __prb_previous_block(po, rb, status);
1010}
1011
eea49cc9 1012static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1013 struct packet_ring_buffer *rb)
1014{
1015 switch (po->tp_version) {
1016 case TPACKET_V1:
1017 case TPACKET_V2:
1018 return packet_increment_head(rb);
1019 case TPACKET_V3:
1020 default:
1021 WARN(1, "TPACKET version not supported.\n");
1022 BUG();
1023 return;
1024 }
1025}
1026
eea49cc9 1027static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1028 struct packet_ring_buffer *rb,
1029 int status)
1030{
1031 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1032 return packet_lookup_frame(po, rb, previous, status);
1033}
1034
eea49cc9 1035static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1036{
1037 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1038}
1039
77f65ebd
WB		 1040static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1041{
1042 struct sock *sk = &po->sk;
1043 bool has_room;
1044
1045 if (po->prot_hook.func != tpacket_rcv)
1046 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1047 <= sk->sk_rcvbuf;
1048
1049 spin_lock(&sk->sk_receive_queue.lock);
1050 if (po->tp_version == TPACKET_V3)
1051 has_room = prb_lookup_block(po, &po->rx_ring,
1052 po->rx_ring.prb_bdqc.kactive_blk_num,
1053 TP_STATUS_KERNEL);
1054 else
1055 has_room = packet_lookup_frame(po, &po->rx_ring,
1056 po->rx_ring.head,
1057 TP_STATUS_KERNEL);
1058 spin_unlock(&sk->sk_receive_queue.lock);
1059
1060 return has_room;
1061}
1062
1da177e4
LT		 1063static void packet_sock_destruct(struct sock *sk)
1064{
ed85b565
RC		 1065 skb_queue_purge(&sk->sk_error_queue);
1066
547b792c
IJ		 1067 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1068 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1069
1070 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1071 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1072 return;
1073 }
1074
17ab56a2 1075 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1076}
1077
dc99f600
DM		 1078static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1079{
1080 int x = atomic_read(&f->rr_cur) + 1;
1081
1082 if (x >= num)
1083 x = 0;
1084
1085 return x;
1086}
1087
77f65ebd
WB		 1088static unsigned int fanout_demux_hash(struct packet_fanout *f,
1089 struct sk_buff *skb,
1090 unsigned int num)
dc99f600 1091{
77f65ebd 1092 return (((u64)skb->rxhash) * num) >> 32;
dc99f600
DM		 1093}
1094
77f65ebd
WB		 1095static unsigned int fanout_demux_lb(struct packet_fanout *f,
1096 struct sk_buff *skb,
1097 unsigned int num)
dc99f600
DM		 1098{
1099 int cur, old;
1100
1101 cur = atomic_read(&f->rr_cur);
1102 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1103 fanout_rr_next(f, num))) != cur)
1104 cur = old;
77f65ebd
WB		 1105 return cur;
1106}
1107
1108static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1109 struct sk_buff *skb,
1110 unsigned int num)
1111{
1112 return smp_processor_id() % num;
dc99f600
DM		 1113}
1114
77f65ebd
WB		 1115static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1116 struct sk_buff *skb,
1117 unsigned int idx, unsigned int skip,
1118 unsigned int num)
95ec3eb4 1119{
77f65ebd 1120 unsigned int i, j;
95ec3eb4 1121
77f65ebd
WB		 1122 i = j = min_t(int, f->next[idx], num - 1);
1123 do {
1124 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1125 if (i != j)
1126 f->next[idx] = i;
1127 return i;
1128 }
1129 if (++i == num)
1130 i = 0;
1131 } while (i != j);
1132
1133 return idx;
1134}
1135
1136static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1137{
1138 return f->flags & (flag >> 8);
95ec3eb4
DM		 1139}
1140
95ec3eb4
DM		 1141static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1142 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1143{
1144 struct packet_fanout *f = pt->af_packet_priv;
1145 unsigned int num = f->num_members;
1146 struct packet_sock *po;
77f65ebd 1147 unsigned int idx;
dc99f600
DM		 1148
1149 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1150 !num) {
1151 kfree_skb(skb);
1152 return 0;
1153 }
1154
95ec3eb4
DM		 1155 switch (f->type) {
1156 case PACKET_FANOUT_HASH:
1157 default:
77f65ebd 1158 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1159 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1160 if (!skb)
1161 return 0;
1162 }
1163 skb_get_rxhash(skb);
77f65ebd 1164 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1165 break;
1166 case PACKET_FANOUT_LB:
77f65ebd 1167 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1168 break;
1169 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1170 idx = fanout_demux_cpu(f, skb, num);
1171 break;
1172 case PACKET_FANOUT_ROLLOVER:
1173 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1174 break;
dc99f600
DM		 1175 }
1176
77f65ebd
WB		 1177 po = pkt_sk(f->arr[idx]);
1178 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1179 unlikely(!packet_rcv_has_room(po, skb))) {
1180 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1181 po = pkt_sk(f->arr[idx]);
1182 }
dc99f600
DM		 1183
1184 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1185}
1186
fff3321d
PE		 1187DEFINE_MUTEX(fanout_mutex);
1188EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1189static LIST_HEAD(fanout_list);
1190
1191static void __fanout_link(struct sock *sk, struct packet_sock *po)
1192{
1193 struct packet_fanout *f = po->fanout;
1194
1195 spin_lock(&f->lock);
1196 f->arr[f->num_members] = sk;
1197 smp_wmb();
1198 f->num_members++;
1199 spin_unlock(&f->lock);
1200}
1201
1202static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1203{
1204 struct packet_fanout *f = po->fanout;
1205 int i;
1206
1207 spin_lock(&f->lock);
1208 for (i = 0; i < f->num_members; i++) {
1209 if (f->arr[i] == sk)
1210 break;
1211 }
1212 BUG_ON(i >= f->num_members);
1213 f->arr[i] = f->arr[f->num_members - 1];
1214 f->num_members--;
1215 spin_unlock(&f->lock);
1216}
1217
a0dfb263 1218static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
c0de08d0
EL		 1219{
1220 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1221 return true;
1222
1223 return false;
1224}
1225
7736d33f 1226static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1227{
1228 struct packet_sock *po = pkt_sk(sk);
1229 struct packet_fanout *f, *match;
7736d33f 1230 u8 type = type_flags & 0xff;
77f65ebd 1231 u8 flags = type_flags >> 8;
dc99f600
DM		 1232 int err;
1233
1234 switch (type) {
77f65ebd
WB		 1235 case PACKET_FANOUT_ROLLOVER:
1236 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1237 return -EINVAL;
dc99f600
DM		 1238 case PACKET_FANOUT_HASH:
1239 case PACKET_FANOUT_LB:
95ec3eb4 1240 case PACKET_FANOUT_CPU:
dc99f600
DM		 1241 break;
1242 default:
1243 return -EINVAL;
1244 }
1245
1246 if (!po->running)
1247 return -EINVAL;
1248
1249 if (po->fanout)
1250 return -EALREADY;
1251
1252 mutex_lock(&fanout_mutex);
1253 match = NULL;
1254 list_for_each_entry(f, &fanout_list, list) {
1255 if (f->id == id &&
1256 read_pnet(&f->net) == sock_net(sk)) {
1257 match = f;
1258 break;
1259 }
1260 }
afe62c68 1261 err = -EINVAL;
77f65ebd 1262 if (match && match->flags != flags)
afe62c68 1263 goto out;
dc99f600 1264 if (!match) {
afe62c68 1265 err = -ENOMEM;
dc99f600 1266 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1267 if (!match)
1268 goto out;
1269 write_pnet(&match->net, sock_net(sk));
1270 match->id = id;
1271 match->type = type;
77f65ebd 1272 match->flags = flags;
afe62c68
ED		 1273 atomic_set(&match->rr_cur, 0);
1274 INIT_LIST_HEAD(&match->list);
1275 spin_lock_init(&match->lock);
1276 atomic_set(&match->sk_ref, 0);
1277 match->prot_hook.type = po->prot_hook.type;
1278 match->prot_hook.dev = po->prot_hook.dev;
1279 match->prot_hook.func = packet_rcv_fanout;
1280 match->prot_hook.af_packet_priv = match;
c0de08d0 1281 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1282 dev_add_pack(&match->prot_hook);
1283 list_add(&match->list, &fanout_list);
dc99f600 1284 }
afe62c68
ED		 1285 err = -EINVAL;
1286 if (match->type == type &&
1287 match->prot_hook.type == po->prot_hook.type &&
1288 match->prot_hook.dev == po->prot_hook.dev) {
1289 err = -ENOSPC;
1290 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1291 __dev_remove_pack(&po->prot_hook);
1292 po->fanout = match;
1293 atomic_inc(&match->sk_ref);
1294 __fanout_link(sk, po);
1295 err = 0;
dc99f600
DM		 1296 }
1297 }
afe62c68 1298out:
dc99f600
DM		 1299 mutex_unlock(&fanout_mutex);
1300 return err;
1301}
1302
1303static void fanout_release(struct sock *sk)
1304{
1305 struct packet_sock *po = pkt_sk(sk);
1306 struct packet_fanout *f;
1307
1308 f = po->fanout;
1309 if (!f)
1310 return;
1311
fff3321d 1312 mutex_lock(&fanout_mutex);
dc99f600
DM		 1313 po->fanout = NULL;
1314
dc99f600
DM		 1315 if (atomic_dec_and_test(&f->sk_ref)) {
1316 list_del(&f->list);
1317 dev_remove_pack(&f->prot_hook);
1318 kfree(f);
1319 }
1320 mutex_unlock(&fanout_mutex);
1321}
1da177e4 1322
90ddc4f0 1323static const struct proto_ops packet_ops;
1da177e4 1324
90ddc4f0 1325static const struct proto_ops packet_ops_spkt;
1da177e4 1326
40d4e3df
ED		 1327static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1328 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1329{
1330 struct sock *sk;
1331 struct sockaddr_pkt *spkt;
1332
1333 /*
1334 * When we registered the protocol we saved the socket in the data
1335 * field for just this event.
1336 */
1337
1338 sk = pt->af_packet_priv;
1ce4f28b 1339
1da177e4
LT		 1340 /*
1341 * Yank back the headers [hope the device set this
1342 * right or kerboom...]
1343 *
1344 * Incoming packets have ll header pulled,
1345 * push it back.
1346 *
98e399f8 1347 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1348 * so that this procedure is noop.
1349 */
1350
1351 if (skb->pkt_type == PACKET_LOOPBACK)
1352 goto out;
1353
09ad9bc7 1354 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1355 goto out;
1356
40d4e3df
ED		 1357 skb = skb_share_check(skb, GFP_ATOMIC);
1358 if (skb == NULL)
1da177e4
LT		 1359 goto oom;
1360
1361 /* drop any routing info */
adf30907 1362 skb_dst_drop(skb);
1da177e4 1363
84531c24
PO		 1364 /* drop conntrack reference */
1365 nf_reset(skb);
1366
ffbc6111 1367 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1368
98e399f8 1369 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1370
1371 /*
1372 * The SOCK_PACKET socket receives _all_ frames.
1373 */
1374
1375 spkt->spkt_family = dev->type;
1376 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1377 spkt->spkt_protocol = skb->protocol;
1378
1379 /*
1380 * Charge the memory to the socket. This is done specifically
1381 * to prevent sockets using all the memory up.
1382 */
1383
40d4e3df 1384 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1385 return 0;
1386
1387out:
1388 kfree_skb(skb);
1389oom:
1390 return 0;
1391}
1392
1393
1394/*
1395 * Output a raw packet to a device layer. This bypasses all the other
1396 * protocol layers and you must therefore supply it with a complete frame
1397 */
1ce4f28b 1398
1da177e4
LT		 1399static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1400 struct msghdr *msg, size_t len)
1401{
1402 struct sock *sk = sock->sk;
40d4e3df 1403 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1404 struct sk_buff *skb = NULL;
1da177e4 1405 struct net_device *dev;
40d4e3df 1406 __be16 proto = 0;
1da177e4 1407 int err;
3bdc0eba 1408 int extra_len = 0;
1ce4f28b 1409
1da177e4 1410 /*
1ce4f28b 1411 * Get and verify the address.
1da177e4
LT		 1412 */
1413
40d4e3df 1414 if (saddr) {
1da177e4 1415 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1416 return -EINVAL;
1417 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1418 proto = saddr->spkt_protocol;
1419 } else
1420 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1421
1422 /*
1ce4f28b 1423 * Find the device first to size check it
1da177e4
LT		 1424 */
1425
de74e92a 1426 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1427retry:
654d1f8a
ED		 1428 rcu_read_lock();
1429 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1430 err = -ENODEV;
1431 if (dev == NULL)
1432 goto out_unlock;
1ce4f28b 1433
d5e76b0a
DM		 1434 err = -ENETDOWN;
1435 if (!(dev->flags & IFF_UP))
1436 goto out_unlock;
1437
1da177e4 1438 /*
40d4e3df
ED		 1439 * You may not queue a frame bigger than the mtu. This is the lowest level
1440 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1441 */
1ce4f28b 1442
3bdc0eba
BG		 1443 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1444 if (!netif_supports_nofcs(dev)) {
1445 err = -EPROTONOSUPPORT;
1446 goto out_unlock;
1447 }
1448 extra_len = 4; /* We're doing our own CRC */
1449 }
1450
1da177e4 1451 err = -EMSGSIZE;
3bdc0eba 1452 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1453 goto out_unlock;
1454
1a35ca80
ED		 1455 if (!skb) {
1456 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1457 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1458 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1459
1460 rcu_read_unlock();
4ce40912 1461 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1462 if (skb == NULL)
1463 return -ENOBUFS;
1464 /* FIXME: Save some space for broken drivers that write a hard
1465 * header at transmission time by themselves. PPP is the notable
1466 * one here. This should really be fixed at the driver level.
1467 */
1468 skb_reserve(skb, reserved);
1469 skb_reset_network_header(skb);
1470
1471 /* Try to align data part correctly */
1472 if (hhlen) {
1473 skb->data -= hhlen;
1474 skb->tail -= hhlen;
1475 if (len < hhlen)
1476 skb_reset_network_header(skb);
1477 }
1478 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1479 if (err)
1480 goto out_free;
1481 goto retry;
1da177e4
LT		 1482 }
1483
3bdc0eba 1484 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1485 /* Earlier code assumed this would be a VLAN pkt,
1486 * double-check this now that we have the actual
1487 * packet in hand.
1488 */
1489 struct ethhdr *ehdr;
1490 skb_reset_mac_header(skb);
1491 ehdr = eth_hdr(skb);
1492 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1493 err = -EMSGSIZE;
1494 goto out_unlock;
1495 }
1496 }
1a35ca80 1497
1da177e4
LT		 1498 skb->protocol = proto;
1499 skb->dev = dev;
1500 skb->priority = sk->sk_priority;
2d37a186 1501 skb->mark = sk->sk_mark;
bf84a010
DB		 1502
1503 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1504
3bdc0eba
BG		 1505 if (unlikely(extra_len == 4))
1506 skb->no_fcs = 1;
1507
40893fd0 1508 skb_probe_transport_header(skb, 0);
c1aad275 1509
1da177e4 1510 dev_queue_xmit(skb);
654d1f8a 1511 rcu_read_unlock();
40d4e3df 1512 return len;
1da177e4 1513
1da177e4 1514out_unlock:
654d1f8a 1515 rcu_read_unlock();
1a35ca80
ED		 1516out_free:
1517 kfree_skb(skb);
1da177e4
LT		 1518 return err;
1519}
1da177e4 1520
eea49cc9 1521static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1522 const struct sock *sk,
dbcb5855 1523 unsigned int res)
1da177e4
LT		 1524{
1525 struct sk_filter *filter;
fda9ef5d 1526
80f8f102
ED		 1527 rcu_read_lock();
1528 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1529 if (filter != NULL)
0a14842f 1530 res = SK_RUN_FILTER(filter, skb);
80f8f102 1531 rcu_read_unlock();
1da177e4 1532
dbcb5855 1533 return res;
1da177e4
LT		 1534}
1535
1536/*
62ab0812
ED		 1537 * This function makes lazy skb cloning in hope that most of packets
1538 * are discarded by BPF.
1539 *
1540 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1541 * and skb->cb are mangled. It works because (and until) packets
1542 * falling here are owned by current CPU. Output packets are cloned
1543 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1544 * sequencially, so that if we return skb to original state on exit,
1545 * we will not harm anyone.
1da177e4
LT		 1546 */
1547
40d4e3df
ED		 1548static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1549 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1550{
1551 struct sock *sk;
1552 struct sockaddr_ll *sll;
1553 struct packet_sock *po;
40d4e3df 1554 u8 *skb_head = skb->data;
1da177e4 1555 int skb_len = skb->len;
dbcb5855 1556 unsigned int snaplen, res;
1da177e4
LT		 1557
1558 if (skb->pkt_type == PACKET_LOOPBACK)
1559 goto drop;
1560
1561 sk = pt->af_packet_priv;
1562 po = pkt_sk(sk);
1563
09ad9bc7 1564 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1565 goto drop;
1566
1da177e4
LT		 1567 skb->dev = dev;
1568
3b04ddde 1569 if (dev->header_ops) {
1da177e4 1570 /* The device has an explicit notion of ll header,
62ab0812
ED		 1571 * exported to higher levels.
1572 *
1573 * Otherwise, the device hides details of its frame
1574 * structure, so that corresponding packet head is
1575 * never delivered to user.
1da177e4
LT		 1576 */
1577 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1578 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1579 else if (skb->pkt_type == PACKET_OUTGOING) {
1580 /* Special case: outgoing packets have ll header at head */
bbe735e4 1581 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1582 }
1583 }
1584
1585 snaplen = skb->len;
1586
dbcb5855
DM		 1587 res = run_filter(skb, sk, snaplen);
1588 if (!res)
fda9ef5d 1589 goto drop_n_restore;
dbcb5855
DM		 1590 if (snaplen > res)
1591 snaplen = res;
1da177e4 1592
0fd7bac6 1593 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1594 goto drop_n_acct;
1595
1596 if (skb_shared(skb)) {
1597 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1598 if (nskb == NULL)
1599 goto drop_n_acct;
1600
1601 if (skb_head != skb->data) {
1602 skb->data = skb_head;
1603 skb->len = skb_len;
1604 }
abc4e4fa 1605 consume_skb(skb);
1da177e4
LT		 1606 skb = nskb;
1607 }
1608
ffbc6111
HX		 1609 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1610 sizeof(skb->cb));
1611
1612 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1613 sll->sll_family = AF_PACKET;
1614 sll->sll_hatype = dev->type;
1615 sll->sll_protocol = skb->protocol;
1616 sll->sll_pkttype = skb->pkt_type;
8032b464 1617 if (unlikely(po->origdev))
80feaacb
PWJ		 1618 sll->sll_ifindex = orig_dev->ifindex;
1619 else
1620 sll->sll_ifindex = dev->ifindex;
1da177e4 1621
b95cce35 1622 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1623
ffbc6111 1624 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1625
1da177e4
LT		 1626 if (pskb_trim(skb, snaplen))
1627 goto drop_n_acct;
1628
1629 skb_set_owner_r(skb, sk);
1630 skb->dev = NULL;
adf30907 1631 skb_dst_drop(skb);
1da177e4 1632
84531c24
PO		 1633 /* drop conntrack reference */
1634 nf_reset(skb);
1635
1da177e4
LT		 1636 spin_lock(&sk->sk_receive_queue.lock);
1637 po->stats.tp_packets++;
3b885787 1638 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1639 __skb_queue_tail(&sk->sk_receive_queue, skb);
1640 spin_unlock(&sk->sk_receive_queue.lock);
1641 sk->sk_data_ready(sk, skb->len);
1642 return 0;
1643
1644drop_n_acct:
7091fbd8
WB		 1645 spin_lock(&sk->sk_receive_queue.lock);
1646 po->stats.tp_drops++;
1647 atomic_inc(&sk->sk_drops);
1648 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1649
1650drop_n_restore:
1651 if (skb_head != skb->data && skb_shared(skb)) {
1652 skb->data = skb_head;
1653 skb->len = skb_len;
1654 }
1655drop:
ead2ceb0 1656 consume_skb(skb);
1da177e4
LT		 1657 return 0;
1658}
1659
40d4e3df
ED		 1660static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1661 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1662{
1663 struct sock *sk;
1664 struct packet_sock *po;
1665 struct sockaddr_ll *sll;
184f489e 1666 union tpacket_uhdr h;
40d4e3df 1667 u8 *skb_head = skb->data;
1da177e4 1668 int skb_len = skb->len;
dbcb5855 1669 unsigned int snaplen, res;
f6fb8f10 1670 unsigned long status = TP_STATUS_USER;
bbd6ef87 1671 unsigned short macoff, netoff, hdrlen;
1da177e4 1672 struct sk_buff *copy_skb = NULL;
b7aa0bf7 1673 struct timeval tv;
bbd6ef87 1674 struct timespec ts;
614f60fa 1675 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
1da177e4
LT		 1676
1677 if (skb->pkt_type == PACKET_LOOPBACK)
1678 goto drop;
1679
1680 sk = pt->af_packet_priv;
1681 po = pkt_sk(sk);
1682
09ad9bc7 1683 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1684 goto drop;
1685
3b04ddde 1686 if (dev->header_ops) {
1da177e4 1687 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1688 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1689 else if (skb->pkt_type == PACKET_OUTGOING) {
1690 /* Special case: outgoing packets have ll header at head */
bbe735e4 1691 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1692 }
1693 }
1694
8dc41944
HX		 1695 if (skb->ip_summed == CHECKSUM_PARTIAL)
1696 status |= TP_STATUS_CSUMNOTREADY;
1697
1da177e4
LT		 1698 snaplen = skb->len;
1699
dbcb5855
DM		 1700 res = run_filter(skb, sk, snaplen);
1701 if (!res)
fda9ef5d 1702 goto drop_n_restore;
dbcb5855
DM		 1703 if (snaplen > res)
1704 snaplen = res;
1da177e4
LT		 1705
1706 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1707 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1708 po->tp_reserve;
1da177e4 1709 } else {
95c96174 1710 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1711 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1712 (maclen < 16 ? 16 : maclen)) +
1713 po->tp_reserve;
1da177e4
LT		 1714 macoff = netoff - maclen;
1715 }
f6fb8f10 1716 if (po->tp_version <= TPACKET_V2) {
1717 if (macoff + snaplen > po->rx_ring.frame_size) {
1718 if (po->copy_thresh &&
0fd7bac6 1719 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1720 if (skb_shared(skb)) {
1721 copy_skb = skb_clone(skb, GFP_ATOMIC);
1722 } else {
1723 copy_skb = skb_get(skb);
1724 skb_head = skb->data;
1725 }
1726 if (copy_skb)
1727 skb_set_owner_r(copy_skb, sk);
1da177e4 1728 }
f6fb8f10 1729 snaplen = po->rx_ring.frame_size - macoff;
1730 if ((int)snaplen < 0)
1731 snaplen = 0;
1da177e4 1732 }
1da177e4 1733 }
1da177e4 1734 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1735 h.raw = packet_current_rx_frame(po, skb,
1736 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1737 if (!h.raw)
1da177e4 1738 goto ring_is_full;
f6fb8f10 1739 if (po->tp_version <= TPACKET_V2) {
1740 packet_increment_rx_head(po, &po->rx_ring);
1741 /*
1742 * LOSING will be reported till you read the stats,
1743 * because it's COR - Clear On Read.
1744 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1745 * at packet level.
1746 */
1747 if (po->stats.tp_drops)
1748 status |= TP_STATUS_LOSING;
1749 }
1da177e4
LT		 1750 po->stats.tp_packets++;
1751 if (copy_skb) {
1752 status |= TP_STATUS_COPY;
1753 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1754 }
1da177e4
LT		 1755 spin_unlock(&sk->sk_receive_queue.lock);
1756
bbd6ef87 1757 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
1da177e4 1758
bbd6ef87
PM		 1759 switch (po->tp_version) {
1760 case TPACKET_V1:
1761 h.h1->tp_len = skb->len;
1762 h.h1->tp_snaplen = snaplen;
1763 h.h1->tp_mac = macoff;
1764 h.h1->tp_net = netoff;
614f60fa
SM		 1765 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1766 && shhwtstamps->syststamp.tv64)
1767 tv = ktime_to_timeval(shhwtstamps->syststamp);
1768 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1769 && shhwtstamps->hwtstamp.tv64)
1770 tv = ktime_to_timeval(shhwtstamps->hwtstamp);
1771 else if (skb->tstamp.tv64)
bbd6ef87
PM		 1772 tv = ktime_to_timeval(skb->tstamp);
1773 else
1774 do_gettimeofday(&tv);
1775 h.h1->tp_sec = tv.tv_sec;
1776 h.h1->tp_usec = tv.tv_usec;
1777 hdrlen = sizeof(*h.h1);
1778 break;
1779 case TPACKET_V2:
1780 h.h2->tp_len = skb->len;
1781 h.h2->tp_snaplen = snaplen;
1782 h.h2->tp_mac = macoff;
1783 h.h2->tp_net = netoff;
614f60fa
SM		 1784 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1785 && shhwtstamps->syststamp.tv64)
1786 ts = ktime_to_timespec(shhwtstamps->syststamp);
1787 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1788 && shhwtstamps->hwtstamp.tv64)
1789 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
1790 else if (skb->tstamp.tv64)
bbd6ef87
PM		 1791 ts = ktime_to_timespec(skb->tstamp);
1792 else
1793 getnstimeofday(&ts);
1794 h.h2->tp_sec = ts.tv_sec;
1795 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1796 if (vlan_tx_tag_present(skb)) {
1797 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1798 status |= TP_STATUS_VLAN_VALID;
1799 } else {
1800 h.h2->tp_vlan_tci = 0;
1801 }
13fcb7bd 1802 h.h2->tp_padding = 0;
bbd6ef87
PM		 1803 hdrlen = sizeof(*h.h2);
1804 break;
f6fb8f10 1805 case TPACKET_V3:
1806 /* tp_nxt_offset,vlan are already populated above.
1807 * So DONT clear those fields here
1808 */
1809 h.h3->tp_status |= status;
1810 h.h3->tp_len = skb->len;
1811 h.h3->tp_snaplen = snaplen;
1812 h.h3->tp_mac = macoff;
1813 h.h3->tp_net = netoff;
1814 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1815 && shhwtstamps->syststamp.tv64)
1816 ts = ktime_to_timespec(shhwtstamps->syststamp);
1817 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1818 && shhwtstamps->hwtstamp.tv64)
1819 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
1820 else if (skb->tstamp.tv64)
1821 ts = ktime_to_timespec(skb->tstamp);
1822 else
1823 getnstimeofday(&ts);
1824 h.h3->tp_sec = ts.tv_sec;
1825 h.h3->tp_nsec = ts.tv_nsec;
1826 hdrlen = sizeof(*h.h3);
1827 break;
bbd6ef87
PM		 1828 default:
1829 BUG();
1830 }
1da177e4 1831
bbd6ef87 1832 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1833 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1834 sll->sll_family = AF_PACKET;
1835 sll->sll_hatype = dev->type;
1836 sll->sll_protocol = skb->protocol;
1837 sll->sll_pkttype = skb->pkt_type;
8032b464 1838 if (unlikely(po->origdev))
80feaacb
PWJ		 1839 sll->sll_ifindex = orig_dev->ifindex;
1840 else
1841 sll->sll_ifindex = dev->ifindex;
1da177e4 1842
e16aa207 1843 smp_mb();
f6dafa95 1844#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1845 {
0af55bb5
CG		 1846 u8 *start, *end;
1847
f6fb8f10 1848 if (po->tp_version <= TPACKET_V2) {
1849 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1850 + macoff + snaplen);
1851 for (start = h.raw; start < end; start += PAGE_SIZE)
1852 flush_dcache_page(pgv_to_page(start));
1853 }
cc9f01b2 1854 smp_wmb();
1da177e4 1855 }
f6dafa95 1856#endif
f6fb8f10 1857 if (po->tp_version <= TPACKET_V2)
1858 __packet_set_status(po, h.raw, status);
1859 else
1860 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1861
1862 sk->sk_data_ready(sk, 0);
1863
1864drop_n_restore:
1865 if (skb_head != skb->data && skb_shared(skb)) {
1866 skb->data = skb_head;
1867 skb->len = skb_len;
1868 }
1869drop:
1ce4f28b 1870 kfree_skb(skb);
1da177e4
LT		 1871 return 0;
1872
1873ring_is_full:
1874 po->stats.tp_drops++;
1875 spin_unlock(&sk->sk_receive_queue.lock);
1876
1877 sk->sk_data_ready(sk, 0);
acb5d75b 1878 kfree_skb(copy_skb);
1da177e4
LT		 1879 goto drop_n_restore;
1880}
1881
69e3c75f
JB		 1882static void tpacket_destruct_skb(struct sk_buff *skb)
1883{
1884 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1885 void *ph;
1da177e4 1886
69e3c75f
JB		 1887 if (likely(po->tx_ring.pg_vec)) {
1888 ph = skb_shinfo(skb)->destructor_arg;
69e3c75f
JB		 1889 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1890 atomic_dec(&po->tx_ring.pending);
1891 __packet_set_status(po, ph, TP_STATUS_AVAILABLE);
1892 }
1893
1894 sock_wfree(skb);
1895}
1896
40d4e3df
ED		 1897static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1898 void *frame, struct net_device *dev, int size_max,
ae641949 1899 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 1900{
184f489e 1901 union tpacket_uhdr ph;
69e3c75f
JB		 1902 int to_write, offset, len, tp_len, nr_frags, len_max;
1903 struct socket *sock = po->sk.sk_socket;
1904 struct page *page;
1905 void *data;
1906 int err;
1907
1908 ph.raw = frame;
1909
1910 skb->protocol = proto;
1911 skb->dev = dev;
1912 skb->priority = po->sk.sk_priority;
2d37a186 1913 skb->mark = po->sk.sk_mark;
69e3c75f
JB		 1914 skb_shinfo(skb)->destructor_arg = ph.raw;
1915
1916 switch (po->tp_version) {
1917 case TPACKET_V2:
1918 tp_len = ph.h2->tp_len;
1919 break;
1920 default:
1921 tp_len = ph.h1->tp_len;
1922 break;
1923 }
1924 if (unlikely(tp_len > size_max)) {
40d4e3df 1925 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
69e3c75f
JB		 1926 return -EMSGSIZE;
1927 }
1928
ae641949 1929 skb_reserve(skb, hlen);
69e3c75f 1930 skb_reset_network_header(skb);
40893fd0 1931 skb_probe_transport_header(skb, 0);
c1aad275 1932
5920cd3a
PC		 1933 if (po->tp_tx_has_off) {
1934 int off_min, off_max, off;
1935 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
1936 off_max = po->tx_ring.frame_size - tp_len;
1937 if (sock->type == SOCK_DGRAM) {
1938 switch (po->tp_version) {
1939 case TPACKET_V2:
1940 off = ph.h2->tp_net;
1941 break;
1942 default:
1943 off = ph.h1->tp_net;
1944 break;
1945 }
1946 } else {
1947 switch (po->tp_version) {
1948 case TPACKET_V2:
1949 off = ph.h2->tp_mac;
1950 break;
1951 default:
1952 off = ph.h1->tp_mac;
1953 break;
1954 }
1955 }
1956 if (unlikely((off < off_min) || (off_max < off)))
1957 return -EINVAL;
1958 data = ph.raw + off;
1959 } else {
1960 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
1961 }
69e3c75f
JB		 1962 to_write = tp_len;
1963
1964 if (sock->type == SOCK_DGRAM) {
1965 err = dev_hard_header(skb, dev, ntohs(proto), addr,
1966 NULL, tp_len);
1967 if (unlikely(err < 0))
1968 return -EINVAL;
40d4e3df 1969 } else if (dev->hard_header_len) {
69e3c75f
JB		 1970 /* net device doesn't like empty head */
1971 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 1972 pr_err("packet size is too short (%d < %d)\n",
1973 tp_len, dev->hard_header_len);
69e3c75f
JB		 1974 return -EINVAL;
1975 }
1976
1977 skb_push(skb, dev->hard_header_len);
1978 err = skb_store_bits(skb, 0, data,
1979 dev->hard_header_len);
1980 if (unlikely(err))
1981 return err;
1982
1983 data += dev->hard_header_len;
1984 to_write -= dev->hard_header_len;
1985 }
1986
69e3c75f
JB		 1987 offset = offset_in_page(data);
1988 len_max = PAGE_SIZE - offset;
1989 len = ((to_write > len_max) ? len_max : to_write);
1990
1991 skb->data_len = to_write;
1992 skb->len += to_write;
1993 skb->truesize += to_write;
1994 atomic_add(to_write, &po->sk.sk_wmem_alloc);
1995
1996 while (likely(to_write)) {
1997 nr_frags = skb_shinfo(skb)->nr_frags;
1998
1999 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2000 pr_err("Packet exceed the number of skb frags(%lu)\n",
2001 MAX_SKB_FRAGS);
69e3c75f
JB		 2002 return -EFAULT;
2003 }
2004
0af55bb5
CG		 2005 page = pgv_to_page(data);
2006 data += len;
69e3c75f
JB		 2007 flush_dcache_page(page);
2008 get_page(page);
0af55bb5 2009 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2010 to_write -= len;
2011 offset = 0;
2012 len_max = PAGE_SIZE;
2013 len = ((to_write > len_max) ? len_max : to_write);
2014 }
2015
2016 return tp_len;
2017}
2018
2019static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2020{
69e3c75f
JB		 2021 struct sk_buff *skb;
2022 struct net_device *dev;
2023 __be16 proto;
827d9780
BG		 2024 bool need_rls_dev = false;
2025 int err, reserve = 0;
40d4e3df
ED		 2026 void *ph;
2027 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2028 int tp_len, size_max;
2029 unsigned char *addr;
2030 int len_sum = 0;
9e67030a 2031 int status = TP_STATUS_AVAILABLE;
ae641949 2032 int hlen, tlen;
69e3c75f 2033
69e3c75f
JB		 2034 mutex_lock(&po->pg_vec_lock);
2035
69e3c75f 2036 if (saddr == NULL) {
827d9780 2037 dev = po->prot_hook.dev;
69e3c75f
JB		 2038 proto = po->num;
2039 addr = NULL;
2040 } else {
2041 err = -EINVAL;
2042 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2043 goto out;
2044 if (msg->msg_namelen < (saddr->sll_halen
2045 + offsetof(struct sockaddr_ll,
2046 sll_addr)))
2047 goto out;
69e3c75f
JB		 2048 proto = saddr->sll_protocol;
2049 addr = saddr->sll_addr;
827d9780
BG		 2050 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2051 need_rls_dev = true;
69e3c75f
JB		 2052 }
2053
69e3c75f
JB		 2054 err = -ENXIO;
2055 if (unlikely(dev == NULL))
2056 goto out;
2057
2058 reserve = dev->hard_header_len;
2059
2060 err = -ENETDOWN;
2061 if (unlikely(!(dev->flags & IFF_UP)))
2062 goto out_put;
2063
2064 size_max = po->tx_ring.frame_size
b5dd884e 2065 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f
JB		 2066
2067 if (size_max > dev->mtu + reserve)
2068 size_max = dev->mtu + reserve;
2069
2070 do {
2071 ph = packet_current_frame(po, &po->tx_ring,
2072 TP_STATUS_SEND_REQUEST);
2073
2074 if (unlikely(ph == NULL)) {
2075 schedule();
2076 continue;
2077 }
2078
2079 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2080 hlen = LL_RESERVED_SPACE(dev);
2081 tlen = dev->needed_tailroom;
69e3c75f 2082 skb = sock_alloc_send_skb(&po->sk,
ae641949 2083 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2084 0, &err);
2085
2086 if (unlikely(skb == NULL))
2087 goto out_status;
2088
2089 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2090 addr, hlen);
69e3c75f
JB		 2091
2092 if (unlikely(tp_len < 0)) {
2093 if (po->tp_loss) {
2094 __packet_set_status(po, ph,
2095 TP_STATUS_AVAILABLE);
2096 packet_increment_head(&po->tx_ring);
2097 kfree_skb(skb);
2098 continue;
2099 } else {
2100 status = TP_STATUS_WRONG_FORMAT;
2101 err = tp_len;
2102 goto out_status;
2103 }
2104 }
2105
2106 skb->destructor = tpacket_destruct_skb;
2107 __packet_set_status(po, ph, TP_STATUS_SENDING);
2108 atomic_inc(&po->tx_ring.pending);
2109
2110 status = TP_STATUS_SEND_REQUEST;
2111 err = dev_queue_xmit(skb);
eb70df13
JP		 2112 if (unlikely(err > 0)) {
2113 err = net_xmit_errno(err);
2114 if (err && __packet_get_status(po, ph) ==
2115 TP_STATUS_AVAILABLE) {
2116 /* skb was destructed already */
2117 skb = NULL;
2118 goto out_status;
2119 }
2120 /*
2121 * skb was dropped but not destructed yet;
2122 * let's treat it like congestion or err < 0
2123 */
2124 err = 0;
2125 }
69e3c75f
JB		 2126 packet_increment_head(&po->tx_ring);
2127 len_sum += tp_len;
f64f9e71
JP		 2128 } while (likely((ph != NULL) ||
2129 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2130 (atomic_read(&po->tx_ring.pending))))
2131 );
69e3c75f
JB		 2132
2133 err = len_sum;
2134 goto out_put;
2135
69e3c75f
JB		 2136out_status:
2137 __packet_set_status(po, ph, status);
2138 kfree_skb(skb);
2139out_put:
827d9780
BG		 2140 if (need_rls_dev)
2141 dev_put(dev);
69e3c75f
JB		 2142out:
2143 mutex_unlock(&po->pg_vec_lock);
2144 return err;
2145}
69e3c75f 2146
eea49cc9
OJ		 2147static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2148 size_t reserve, size_t len,
2149 size_t linear, int noblock,
2150 int *err)
bfd5f4a3
SS		 2151{
2152 struct sk_buff *skb;
2153
2154 /* Under a page? Don't bother with paged skb. */
2155 if (prepad + len < PAGE_SIZE || !linear)
2156 linear = len;
2157
2158 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2159 err);
2160 if (!skb)
2161 return NULL;
2162
2163 skb_reserve(skb, reserve);
2164 skb_put(skb, linear);
2165 skb->data_len = len - linear;
2166 skb->len += len - linear;
2167
2168 return skb;
2169}
2170
69e3c75f 2171static int packet_snd(struct socket *sock,
1da177e4
LT		 2172 struct msghdr *msg, size_t len)
2173{
2174 struct sock *sk = sock->sk;
40d4e3df 2175 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2176 struct sk_buff *skb;
2177 struct net_device *dev;
0e11c91e 2178 __be16 proto;
827d9780 2179 bool need_rls_dev = false;
1da177e4 2180 unsigned char *addr;
827d9780 2181 int err, reserve = 0;
bfd5f4a3
SS		 2182 struct virtio_net_hdr vnet_hdr = { 0 };
2183 int offset = 0;
2184 int vnet_hdr_len;
2185 struct packet_sock *po = pkt_sk(sk);
2186 unsigned short gso_type = 0;
ae641949 2187 int hlen, tlen;
3bdc0eba 2188 int extra_len = 0;
1da177e4
LT		 2189
2190 /*
1ce4f28b 2191 * Get and verify the address.
1da177e4 2192 */
1ce4f28b 2193
1da177e4 2194 if (saddr == NULL) {
827d9780 2195 dev = po->prot_hook.dev;
1da177e4
LT		 2196 proto = po->num;
2197 addr = NULL;
2198 } else {
2199 err = -EINVAL;
2200 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2201 goto out;
0fb375fb
EB		 2202 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2203 goto out;
1da177e4
LT		 2204 proto = saddr->sll_protocol;
2205 addr = saddr->sll_addr;
827d9780
BG		 2206 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2207 need_rls_dev = true;
1da177e4
LT		 2208 }
2209
1da177e4
LT		 2210 err = -ENXIO;
2211 if (dev == NULL)
2212 goto out_unlock;
2213 if (sock->type == SOCK_RAW)
2214 reserve = dev->hard_header_len;
2215
d5e76b0a
DM		 2216 err = -ENETDOWN;
2217 if (!(dev->flags & IFF_UP))
2218 goto out_unlock;
2219
bfd5f4a3
SS		 2220 if (po->has_vnet_hdr) {
2221 vnet_hdr_len = sizeof(vnet_hdr);
2222
2223 err = -EINVAL;
2224 if (len < vnet_hdr_len)
2225 goto out_unlock;
2226
2227 len -= vnet_hdr_len;
2228
2229 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2230 vnet_hdr_len);
2231 if (err < 0)
2232 goto out_unlock;
2233
2234 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2235 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2236 vnet_hdr.hdr_len))
2237 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2238 vnet_hdr.csum_offset + 2;
2239
2240 err = -EINVAL;
2241 if (vnet_hdr.hdr_len > len)
2242 goto out_unlock;
2243
2244 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2245 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2246 case VIRTIO_NET_HDR_GSO_TCPV4:
2247 gso_type = SKB_GSO_TCPV4;
2248 break;
2249 case VIRTIO_NET_HDR_GSO_TCPV6:
2250 gso_type = SKB_GSO_TCPV6;
2251 break;
2252 case VIRTIO_NET_HDR_GSO_UDP:
2253 gso_type = SKB_GSO_UDP;
2254 break;
2255 default:
2256 goto out_unlock;
2257 }
2258
2259 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2260 gso_type |= SKB_GSO_TCP_ECN;
2261
2262 if (vnet_hdr.gso_size == 0)
2263 goto out_unlock;
2264
2265 }
2266 }
2267
3bdc0eba
BG		 2268 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2269 if (!netif_supports_nofcs(dev)) {
2270 err = -EPROTONOSUPPORT;
2271 goto out_unlock;
2272 }
2273 extra_len = 4; /* We're doing our own CRC */
2274 }
2275
1da177e4 2276 err = -EMSGSIZE;
3bdc0eba 2277 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2278 goto out_unlock;
2279
bfd5f4a3 2280 err = -ENOBUFS;
ae641949
HX		 2281 hlen = LL_RESERVED_SPACE(dev);
2282 tlen = dev->needed_tailroom;
2283 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2284 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2285 if (skb == NULL)
1da177e4
LT		 2286 goto out_unlock;
2287
bfd5f4a3 2288 skb_set_network_header(skb, reserve);
1da177e4 2289
0c4e8581
SH		 2290 err = -EINVAL;
2291 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2292 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2293 goto out_free;
1da177e4
LT		 2294
2295 /* Returns -EFAULT on error */
bfd5f4a3 2296 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2297 if (err)
2298 goto out_free;
bf84a010
DB		 2299
2300 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2301
3bdc0eba 2302 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
57f89bfa
BG		 2303 /* Earlier code assumed this would be a VLAN pkt,
2304 * double-check this now that we have the actual
2305 * packet in hand.
2306 */
2307 struct ethhdr *ehdr;
2308 skb_reset_mac_header(skb);
2309 ehdr = eth_hdr(skb);
2310 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2311 err = -EMSGSIZE;
2312 goto out_free;
2313 }
2314 }
2315
1da177e4
LT		 2316 skb->protocol = proto;
2317 skb->dev = dev;
2318 skb->priority = sk->sk_priority;
2d37a186 2319 skb->mark = sk->sk_mark;
1da177e4 2320
bfd5f4a3
SS		 2321 if (po->has_vnet_hdr) {
2322 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2323 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2324 vnet_hdr.csum_offset)) {
2325 err = -EINVAL;
2326 goto out_free;
2327 }
2328 }
2329
2330 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2331 skb_shinfo(skb)->gso_type = gso_type;
2332
2333 /* Header must be checked, and gso_segs computed. */
2334 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2335 skb_shinfo(skb)->gso_segs = 0;
2336
2337 len += vnet_hdr_len;
2338 }
2339
40893fd0 2340 skb_probe_transport_header(skb, reserve);
c1aad275 2341
3bdc0eba
BG		 2342 if (unlikely(extra_len == 4))
2343 skb->no_fcs = 1;
2344
1da177e4
LT		 2345 /*
2346 * Now send it
2347 */
2348
2349 err = dev_queue_xmit(skb);
2350 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2351 goto out_unlock;
2352
827d9780
BG		 2353 if (need_rls_dev)
2354 dev_put(dev);
1da177e4 2355
40d4e3df 2356 return len;
1da177e4
LT		 2357
2358out_free:
2359 kfree_skb(skb);
2360out_unlock:
827d9780 2361 if (dev && need_rls_dev)
1da177e4
LT		 2362 dev_put(dev);
2363out:
2364 return err;
2365}
2366
69e3c75f
JB		 2367static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2368 struct msghdr *msg, size_t len)
2369{
69e3c75f
JB		 2370 struct sock *sk = sock->sk;
2371 struct packet_sock *po = pkt_sk(sk);
2372 if (po->tx_ring.pg_vec)
2373 return tpacket_snd(po, msg);
2374 else
69e3c75f
JB		 2375 return packet_snd(sock, msg, len);
2376}
2377
1da177e4
LT		 2378/*
2379 * Close a PACKET socket. This is fairly simple. We immediately go
2380 * to 'closed' state and remove our protocol entry in the device list.
2381 */
2382
2383static int packet_release(struct socket *sock)
2384{
2385 struct sock *sk = sock->sk;
2386 struct packet_sock *po;
d12d01d6 2387 struct net *net;
f6fb8f10 2388 union tpacket_req_u req_u;
1da177e4
LT		 2389
2390 if (!sk)
2391 return 0;
2392
3b1e0a65 2393 net = sock_net(sk);
1da177e4
LT		 2394 po = pkt_sk(sk);
2395
0fa7fa98 2396 mutex_lock(&net->packet.sklist_lock);
808f5114 2397 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2398 mutex_unlock(&net->packet.sklist_lock);
2399
2400 preempt_disable();
920de804 2401 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2402 preempt_enable();
1da177e4 2403
808f5114 2404 spin_lock(&po->bind_lock);
ce06b03e 2405 unregister_prot_hook(sk, false);
160ff18a
BG		 2406 if (po->prot_hook.dev) {
2407 dev_put(po->prot_hook.dev);
2408 po->prot_hook.dev = NULL;
2409 }
808f5114 2410 spin_unlock(&po->bind_lock);
1da177e4 2411
1da177e4 2412 packet_flush_mclist(sk);
1da177e4 2413
9665d5d6
PS		 2414 if (po->rx_ring.pg_vec) {
2415 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2416 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2417 }
69e3c75f 2418
9665d5d6
PS		 2419 if (po->tx_ring.pg_vec) {
2420 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2421 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2422 }
1da177e4 2423
dc99f600
DM		 2424 fanout_release(sk);
2425
808f5114 2426 synchronize_net();
1da177e4
LT		 2427 /*
2428 * Now the socket is dead. No more input will appear.
2429 */
1da177e4
LT		 2430 sock_orphan(sk);
2431 sock->sk = NULL;
2432
2433 /* Purge queues */
2434
2435 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2436 sk_refcnt_debug_release(sk);
1da177e4
LT		 2437
2438 sock_put(sk);
2439 return 0;
2440}
2441
2442/*
2443 * Attach a packet hook.
2444 */
2445
0e11c91e 2446static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2447{
2448 struct packet_sock *po = pkt_sk(sk);
dc99f600 2449
aef950b4
WY		 2450 if (po->fanout) {
2451 if (dev)
2452 dev_put(dev);
2453
dc99f600 2454 return -EINVAL;
aef950b4 2455 }
1da177e4
LT		 2456
2457 lock_sock(sk);
2458
2459 spin_lock(&po->bind_lock);
ce06b03e 2460 unregister_prot_hook(sk, true);
1da177e4
LT		 2461 po->num = protocol;
2462 po->prot_hook.type = protocol;
160ff18a
BG		 2463 if (po->prot_hook.dev)
2464 dev_put(po->prot_hook.dev);
1da177e4
LT		 2465 po->prot_hook.dev = dev;
2466
2467 po->ifindex = dev ? dev->ifindex : 0;
2468
2469 if (protocol == 0)
2470 goto out_unlock;
2471
be85d4ad 2472 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2473 register_prot_hook(sk);
be85d4ad
UT		 2474 } else {
2475 sk->sk_err = ENETDOWN;
2476 if (!sock_flag(sk, SOCK_DEAD))
2477 sk->sk_error_report(sk);
1da177e4
LT		 2478 }
2479
2480out_unlock:
2481 spin_unlock(&po->bind_lock);
2482 release_sock(sk);
2483 return 0;
2484}
2485
2486/*
2487 * Bind a packet socket to a device
2488 */
2489
40d4e3df
ED		 2490static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2491 int addr_len)
1da177e4 2492{
40d4e3df 2493 struct sock *sk = sock->sk;
1da177e4
LT		 2494 char name[15];
2495 struct net_device *dev;
2496 int err = -ENODEV;
1ce4f28b 2497
1da177e4
LT		 2498 /*
2499 * Check legality
2500 */
1ce4f28b 2501
8ae55f04 2502 if (addr_len != sizeof(struct sockaddr))
1da177e4 2503 return -EINVAL;
40d4e3df 2504 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2505
3b1e0a65 2506 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2507 if (dev)
1da177e4 2508 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2509 return err;
2510}
1da177e4
LT		 2511
2512static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2513{
40d4e3df
ED		 2514 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2515 struct sock *sk = sock->sk;
1da177e4
LT		 2516 struct net_device *dev = NULL;
2517 int err;
2518
2519
2520 /*
2521 * Check legality
2522 */
1ce4f28b 2523
1da177e4
LT		 2524 if (addr_len < sizeof(struct sockaddr_ll))
2525 return -EINVAL;
2526 if (sll->sll_family != AF_PACKET)
2527 return -EINVAL;
2528
2529 if (sll->sll_ifindex) {
2530 err = -ENODEV;
3b1e0a65 2531 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2532 if (dev == NULL)
2533 goto out;
2534 }
2535 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2536
2537out:
2538 return err;
2539}
2540
2541static struct proto packet_proto = {
2542 .name = "PACKET",
2543 .owner = THIS_MODULE,
2544 .obj_size = sizeof(struct packet_sock),
2545};
2546
2547/*
1ce4f28b 2548 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2549 */
2550
3f378b68
EP		 2551static int packet_create(struct net *net, struct socket *sock, int protocol,
2552 int kern)
1da177e4
LT		 2553{
2554 struct sock *sk;
2555 struct packet_sock *po;
0e11c91e 2556 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2557 int err;
2558
df008c91 2559 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2560 return -EPERM;
be02097c
DM		 2561 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2562 sock->type != SOCK_PACKET)
1da177e4
LT		 2563 return -ESOCKTNOSUPPORT;
2564
2565 sock->state = SS_UNCONNECTED;
2566
2567 err = -ENOBUFS;
6257ff21 2568 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2569 if (sk == NULL)
2570 goto out;
2571
2572 sock->ops = &packet_ops;
1da177e4
LT		 2573 if (sock->type == SOCK_PACKET)
2574 sock->ops = &packet_ops_spkt;
be02097c 2575
1da177e4
LT		 2576 sock_init_data(sock, sk);
2577
2578 po = pkt_sk(sk);
2579 sk->sk_family = PF_PACKET;
0e11c91e 2580 po->num = proto;
1da177e4
LT		 2581
2582 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2583 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2584
2585 /*
2586 * Attach a protocol block
2587 */
2588
2589 spin_lock_init(&po->bind_lock);
905db440 2590 mutex_init(&po->pg_vec_lock);
1da177e4 2591 po->prot_hook.func = packet_rcv;
be02097c 2592
1da177e4
LT		 2593 if (sock->type == SOCK_PACKET)
2594 po->prot_hook.func = packet_rcv_spkt;
be02097c 2595
1da177e4
LT		 2596 po->prot_hook.af_packet_priv = sk;
2597
0e11c91e
AV		 2598 if (proto) {
2599 po->prot_hook.type = proto;
ce06b03e 2600 register_prot_hook(sk);
1da177e4
LT		 2601 }
2602
0fa7fa98 2603 mutex_lock(&net->packet.sklist_lock);
808f5114 2604 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2605 mutex_unlock(&net->packet.sklist_lock);
2606
2607 preempt_disable();
3680453c 2608 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2609 preempt_enable();
808f5114 2610
40d4e3df 2611 return 0;
1da177e4
LT		 2612out:
2613 return err;
2614}
2615
ed85b565
RC		 2616static int packet_recv_error(struct sock *sk, struct msghdr *msg, int len)
2617{
2618 struct sock_exterr_skb *serr;
2619 struct sk_buff *skb, *skb2;
2620 int copied, err;
2621
2622 err = -EAGAIN;
2623 skb = skb_dequeue(&sk->sk_error_queue);
2624 if (skb == NULL)
2625 goto out;
2626
2627 copied = skb->len;
2628 if (copied > len) {
2629 msg->msg_flags |= MSG_TRUNC;
2630 copied = len;
2631 }
2632 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2633 if (err)
2634 goto out_free_skb;
2635
2636 sock_recv_timestamp(msg, sk, skb);
2637
2638 serr = SKB_EXT_ERR(skb);
2639 put_cmsg(msg, SOL_PACKET, PACKET_TX_TIMESTAMP,
2640 sizeof(serr->ee), &serr->ee);
2641
2642 msg->msg_flags |= MSG_ERRQUEUE;
2643 err = copied;
2644
2645 /* Reset and regenerate socket error */
2646 spin_lock_bh(&sk->sk_error_queue.lock);
2647 sk->sk_err = 0;
2648 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) {
2649 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno;
2650 spin_unlock_bh(&sk->sk_error_queue.lock);
2651 sk->sk_error_report(sk);
2652 } else
2653 spin_unlock_bh(&sk->sk_error_queue.lock);
2654
2655out_free_skb:
2656 kfree_skb(skb);
2657out:
2658 return err;
2659}
2660
1da177e4
LT		 2661/*
2662 * Pull a packet from our receive queue and hand it to the user.
2663 * If necessary we block.
2664 */
2665
2666static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2667 struct msghdr *msg, size_t len, int flags)
2668{
2669 struct sock *sk = sock->sk;
2670 struct sk_buff *skb;
2671 int copied, err;
0fb375fb 2672 struct sockaddr_ll *sll;
bfd5f4a3 2673 int vnet_hdr_len = 0;
1da177e4
LT		 2674
2675 err = -EINVAL;
ed85b565 2676 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2677 goto out;
2678
2679#if 0
2680 /* What error should we return now? EUNATTACH? */
2681 if (pkt_sk(sk)->ifindex < 0)
2682 return -ENODEV;
2683#endif
2684
ed85b565
RC		 2685 if (flags & MSG_ERRQUEUE) {
2686 err = packet_recv_error(sk, msg, len);
2687 goto out;
2688 }
2689
1da177e4
LT		 2690 /*
2691 * Call the generic datagram receiver. This handles all sorts
2692 * of horrible races and re-entrancy so we can forget about it
2693 * in the protocol layers.
2694 *
2695 * Now it will return ENETDOWN, if device have just gone down,
2696 * but then it will block.
2697 */
2698
40d4e3df 2699 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2700
2701 /*
1ce4f28b 2702 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2703 * handles the blocking we don't see and worry about blocking
2704 * retries.
2705 */
2706
8ae55f04 2707 if (skb == NULL)
1da177e4
LT		 2708 goto out;
2709
bfd5f4a3
SS		 2710 if (pkt_sk(sk)->has_vnet_hdr) {
2711 struct virtio_net_hdr vnet_hdr = { 0 };
2712
2713 err = -EINVAL;
2714 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2715 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2716 goto out_free;
2717
1f18b717
MK		 2718 len -= vnet_hdr_len;
2719
bfd5f4a3
SS		 2720 if (skb_is_gso(skb)) {
2721 struct skb_shared_info *sinfo = skb_shinfo(skb);
2722
2723 /* This is a hint as to how much should be linear. */
2724 vnet_hdr.hdr_len = skb_headlen(skb);
2725 vnet_hdr.gso_size = sinfo->gso_size;
2726 if (sinfo->gso_type & SKB_GSO_TCPV4)
2727 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2728 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2729 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2730 else if (sinfo->gso_type & SKB_GSO_UDP)
2731 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2732 else if (sinfo->gso_type & SKB_GSO_FCOE)
2733 goto out_free;
2734 else
2735 BUG();
2736 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2737 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2738 } else
2739 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2740
2741 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2742 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2743 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2744 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2745 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2746 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2747 } /* else everything is zero */
2748
2749 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2750 vnet_hdr_len);
2751 if (err < 0)
2752 goto out_free;
2753 }
2754
0fb375fb
EB		 2755 /*
2756 * If the address length field is there to be filled in, we fill
2757 * it in now.
2758 */
2759
ffbc6111 2760 sll = &PACKET_SKB_CB(skb)->sa.ll;
0fb375fb
EB		 2761 if (sock->type == SOCK_PACKET)
2762 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2763 else
2764 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
2765
1da177e4
LT		 2766 /*
2767 * You lose any data beyond the buffer you gave. If it worries a
2768 * user program they can ask the device for its MTU anyway.
2769 */
2770
2771 copied = skb->len;
40d4e3df
ED		 2772 if (copied > len) {
2773 copied = len;
2774 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2775 }
2776
2777 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2778 if (err)
2779 goto out_free;
2780
3b885787 2781 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4
LT		 2782
2783 if (msg->msg_name)
ffbc6111
HX		 2784 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2785 msg->msg_namelen);
1da177e4 2786
8dc41944 2787 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2788 struct tpacket_auxdata aux;
2789
2790 aux.tp_status = TP_STATUS_USER;
2791 if (skb->ip_summed == CHECKSUM_PARTIAL)
2792 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2793 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2794 aux.tp_snaplen = skb->len;
2795 aux.tp_mac = 0;
bbe735e4 2796 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2797 if (vlan_tx_tag_present(skb)) {
2798 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2799 aux.tp_status |= TP_STATUS_VLAN_VALID;
2800 } else {
2801 aux.tp_vlan_tci = 0;
2802 }
13fcb7bd 2803 aux.tp_padding = 0;
ffbc6111 2804 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2805 }
2806
1da177e4
LT		 2807 /*
2808 * Free or return the buffer as appropriate. Again this
2809 * hides all the races and re-entrancy issues from us.
2810 */
bfd5f4a3 2811 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2812
2813out_free:
2814 skb_free_datagram(sk, skb);
2815out:
2816 return err;
2817}
2818
1da177e4
LT		 2819static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2820 int *uaddr_len, int peer)
2821{
2822 struct net_device *dev;
2823 struct sock *sk = sock->sk;
2824
2825 if (peer)
2826 return -EOPNOTSUPP;
2827
2828 uaddr->sa_family = AF_PACKET;
654d1f8a
ED		 2829 rcu_read_lock();
2830 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2831 if (dev)
67286640 2832 strncpy(uaddr->sa_data, dev->name, 14);
654d1f8a 2833 else
1da177e4 2834 memset(uaddr->sa_data, 0, 14);
654d1f8a 2835 rcu_read_unlock();
1da177e4
LT		 2836 *uaddr_len = sizeof(*uaddr);
2837
2838 return 0;
2839}
1da177e4
LT		 2840
2841static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2842 int *uaddr_len, int peer)
2843{
2844 struct net_device *dev;
2845 struct sock *sk = sock->sk;
2846 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2847 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2848
2849 if (peer)
2850 return -EOPNOTSUPP;
2851
2852 sll->sll_family = AF_PACKET;
2853 sll->sll_ifindex = po->ifindex;
2854 sll->sll_protocol = po->num;
67286640 2855 sll->sll_pkttype = 0;
654d1f8a
ED		 2856 rcu_read_lock();
2857 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2858 if (dev) {
2859 sll->sll_hatype = dev->type;
2860 sll->sll_halen = dev->addr_len;
2861 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2862 } else {
2863 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2864 sll->sll_halen = 0;
2865 }
654d1f8a 2866 rcu_read_unlock();
0fb375fb 2867 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2868
2869 return 0;
2870}
2871
2aeb0b88
WC		 2872static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2873 int what)
1da177e4
LT		 2874{
2875 switch (i->type) {
2876 case PACKET_MR_MULTICAST:
1162563f
JP		 2877 if (i->alen != dev->addr_len)
2878 return -EINVAL;
1da177e4 2879 if (what > 0)
22bedad3 2880 return dev_mc_add(dev, i->addr);
1da177e4 2881 else
22bedad3 2882 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2883 break;
2884 case PACKET_MR_PROMISC:
2aeb0b88 2885 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2886 break;
2887 case PACKET_MR_ALLMULTI:
2aeb0b88 2888 return dev_set_allmulti(dev, what);
1da177e4 2889 break;
d95ed927 2890 case PACKET_MR_UNICAST:
1162563f
JP		 2891 if (i->alen != dev->addr_len)
2892 return -EINVAL;
d95ed927 2893 if (what > 0)
a748ee24 2894 return dev_uc_add(dev, i->addr);
d95ed927 2895 else
a748ee24 2896 return dev_uc_del(dev, i->addr);
d95ed927 2897 break;
40d4e3df
ED		 2898 default:
2899 break;
1da177e4 2900 }
2aeb0b88 2901 return 0;
1da177e4
LT		 2902}
2903
2904static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2905{
40d4e3df 2906 for ( ; i; i = i->next) {
1da177e4
LT		 2907 if (i->ifindex == dev->ifindex)
2908 packet_dev_mc(dev, i, what);
2909 }
2910}
2911
0fb375fb 2912static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2913{
2914 struct packet_sock *po = pkt_sk(sk);
2915 struct packet_mclist *ml, *i;
2916 struct net_device *dev;
2917 int err;
2918
2919 rtnl_lock();
2920
2921 err = -ENODEV;
3b1e0a65 2922 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2923 if (!dev)
2924 goto done;
2925
2926 err = -EINVAL;
1162563f 2927 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2928 goto done;
2929
2930 err = -ENOBUFS;
8b3a7005 2931 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 2932 if (i == NULL)
2933 goto done;
2934
2935 err = 0;
2936 for (ml = po->mclist; ml; ml = ml->next) {
2937 if (ml->ifindex == mreq->mr_ifindex &&
2938 ml->type == mreq->mr_type &&
2939 ml->alen == mreq->mr_alen &&
2940 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2941 ml->count++;
2942 /* Free the new element ... */
2943 kfree(i);
2944 goto done;
2945 }
2946 }
2947
2948 i->type = mreq->mr_type;
2949 i->ifindex = mreq->mr_ifindex;
2950 i->alen = mreq->mr_alen;
2951 memcpy(i->addr, mreq->mr_address, i->alen);
2952 i->count = 1;
2953 i->next = po->mclist;
2954 po->mclist = i;
2aeb0b88
WC		 2955 err = packet_dev_mc(dev, i, 1);
2956 if (err) {
2957 po->mclist = i->next;
2958 kfree(i);
2959 }
1da177e4
LT		 2960
2961done:
2962 rtnl_unlock();
2963 return err;
2964}
2965
0fb375fb 2966static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2967{
2968 struct packet_mclist *ml, **mlp;
2969
2970 rtnl_lock();
2971
2972 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
2973 if (ml->ifindex == mreq->mr_ifindex &&
2974 ml->type == mreq->mr_type &&
2975 ml->alen == mreq->mr_alen &&
2976 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2977 if (--ml->count == 0) {
2978 struct net_device *dev;
2979 *mlp = ml->next;
ad959e76
ED		 2980 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2981 if (dev)
1da177e4 2982 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 2983 kfree(ml);
2984 }
2985 rtnl_unlock();
2986 return 0;
2987 }
2988 }
2989 rtnl_unlock();
2990 return -EADDRNOTAVAIL;
2991}
2992
2993static void packet_flush_mclist(struct sock *sk)
2994{
2995 struct packet_sock *po = pkt_sk(sk);
2996 struct packet_mclist *ml;
2997
2998 if (!po->mclist)
2999 return;
3000
3001 rtnl_lock();
3002 while ((ml = po->mclist) != NULL) {
3003 struct net_device *dev;
3004
3005 po->mclist = ml->next;
ad959e76
ED		 3006 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3007 if (dev != NULL)
1da177e4 3008 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3009 kfree(ml);
3010 }
3011 rtnl_unlock();
3012}
1da177e4
LT		 3013
3014static int
b7058842 3015packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3016{
3017 struct sock *sk = sock->sk;
8dc41944 3018 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3019 int ret;
3020
3021 if (level != SOL_PACKET)
3022 return -ENOPROTOOPT;
3023
69e3c75f 3024 switch (optname) {
1ce4f28b 3025 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3026 case PACKET_DROP_MEMBERSHIP:
3027 {
0fb375fb
EB		 3028 struct packet_mreq_max mreq;
3029 int len = optlen;
3030 memset(&mreq, 0, sizeof(mreq));
3031 if (len < sizeof(struct packet_mreq))
1da177e4 3032 return -EINVAL;
0fb375fb
EB		 3033 if (len > sizeof(mreq))
3034 len = sizeof(mreq);
40d4e3df 3035 if (copy_from_user(&mreq, optval, len))
1da177e4 3036 return -EFAULT;
0fb375fb
EB		 3037 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3038 return -EINVAL;
1da177e4
LT		 3039 if (optname == PACKET_ADD_MEMBERSHIP)
3040 ret = packet_mc_add(sk, &mreq);
3041 else
3042 ret = packet_mc_drop(sk, &mreq);
3043 return ret;
3044 }
a2efcfa0 3045
1da177e4 3046 case PACKET_RX_RING:
69e3c75f 3047 case PACKET_TX_RING:
1da177e4 3048 {
f6fb8f10 3049 union tpacket_req_u req_u;
3050 int len;
1da177e4 3051
f6fb8f10 3052 switch (po->tp_version) {
3053 case TPACKET_V1:
3054 case TPACKET_V2:
3055 len = sizeof(req_u.req);
3056 break;
3057 case TPACKET_V3:
3058 default:
3059 len = sizeof(req_u.req3);
3060 break;
3061 }
3062 if (optlen < len)
1da177e4 3063 return -EINVAL;
bfd5f4a3
SS		 3064 if (pkt_sk(sk)->has_vnet_hdr)
3065 return -EINVAL;
f6fb8f10 3066 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3067 return -EFAULT;
f6fb8f10 3068 return packet_set_ring(sk, &req_u, 0,
3069 optname == PACKET_TX_RING);
1da177e4
LT		 3070 }
3071 case PACKET_COPY_THRESH:
3072 {
3073 int val;
3074
40d4e3df 3075 if (optlen != sizeof(val))
1da177e4 3076 return -EINVAL;
40d4e3df 3077 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3078 return -EFAULT;
3079
3080 pkt_sk(sk)->copy_thresh = val;
3081 return 0;
3082 }
bbd6ef87
PM		 3083 case PACKET_VERSION:
3084 {
3085 int val;
3086
3087 if (optlen != sizeof(val))
3088 return -EINVAL;
69e3c75f 3089 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3090 return -EBUSY;
3091 if (copy_from_user(&val, optval, sizeof(val)))
3092 return -EFAULT;
3093 switch (val) {
3094 case TPACKET_V1:
3095 case TPACKET_V2:
f6fb8f10 3096 case TPACKET_V3:
bbd6ef87
PM		 3097 po->tp_version = val;
3098 return 0;
3099 default:
3100 return -EINVAL;
3101 }
3102 }
8913336a
PM		 3103 case PACKET_RESERVE:
3104 {
3105 unsigned int val;
3106
3107 if (optlen != sizeof(val))
3108 return -EINVAL;
69e3c75f 3109 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3110 return -EBUSY;
3111 if (copy_from_user(&val, optval, sizeof(val)))
3112 return -EFAULT;
3113 po->tp_reserve = val;
3114 return 0;
3115 }
69e3c75f
JB		 3116 case PACKET_LOSS:
3117 {
3118 unsigned int val;
3119
3120 if (optlen != sizeof(val))
3121 return -EINVAL;
3122 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3123 return -EBUSY;
3124 if (copy_from_user(&val, optval, sizeof(val)))
3125 return -EFAULT;
3126 po->tp_loss = !!val;
3127 return 0;
3128 }
8dc41944
HX		 3129 case PACKET_AUXDATA:
3130 {
3131 int val;
3132
3133 if (optlen < sizeof(val))
3134 return -EINVAL;
3135 if (copy_from_user(&val, optval, sizeof(val)))
3136 return -EFAULT;
3137
3138 po->auxdata = !!val;
3139 return 0;
3140 }
80feaacb
PWJ		 3141 case PACKET_ORIGDEV:
3142 {
3143 int val;
3144
3145 if (optlen < sizeof(val))
3146 return -EINVAL;
3147 if (copy_from_user(&val, optval, sizeof(val)))
3148 return -EFAULT;
3149
3150 po->origdev = !!val;
3151 return 0;
3152 }
bfd5f4a3
SS		 3153 case PACKET_VNET_HDR:
3154 {
3155 int val;
3156
3157 if (sock->type != SOCK_RAW)
3158 return -EINVAL;
3159 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3160 return -EBUSY;
3161 if (optlen < sizeof(val))
3162 return -EINVAL;
3163 if (copy_from_user(&val, optval, sizeof(val)))
3164 return -EFAULT;
3165
3166 po->has_vnet_hdr = !!val;
3167 return 0;
3168 }
614f60fa
SM		 3169 case PACKET_TIMESTAMP:
3170 {
3171 int val;
3172
3173 if (optlen != sizeof(val))
3174 return -EINVAL;
3175 if (copy_from_user(&val, optval, sizeof(val)))
3176 return -EFAULT;
3177
3178 po->tp_tstamp = val;
3179 return 0;
3180 }
dc99f600
DM		 3181 case PACKET_FANOUT:
3182 {
3183 int val;
3184
3185 if (optlen != sizeof(val))
3186 return -EINVAL;
3187 if (copy_from_user(&val, optval, sizeof(val)))
3188 return -EFAULT;
3189
3190 return fanout_add(sk, val & 0xffff, val >> 16);
3191 }
5920cd3a
PC		 3192 case PACKET_TX_HAS_OFF:
3193 {
3194 unsigned int val;
3195
3196 if (optlen != sizeof(val))
3197 return -EINVAL;
3198 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3199 return -EBUSY;
3200 if (copy_from_user(&val, optval, sizeof(val)))
3201 return -EFAULT;
3202 po->tp_tx_has_off = !!val;
3203 return 0;
3204 }
1da177e4
LT		 3205 default:
3206 return -ENOPROTOOPT;
3207 }
3208}
3209
3210static int packet_getsockopt(struct socket *sock, int level, int optname,
3211 char __user *optval, int __user *optlen)
3212{
3213 int len;
c06fff6e 3214 int val, lv = sizeof(val);
1da177e4
LT		 3215 struct sock *sk = sock->sk;
3216 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3217 void *data = &val;
8dc41944 3218 struct tpacket_stats st;
f6fb8f10 3219 union tpacket_stats_u st_u;
1da177e4
LT		 3220
3221 if (level != SOL_PACKET)
3222 return -ENOPROTOOPT;
3223
8ae55f04
KK		 3224 if (get_user(len, optlen))
3225 return -EFAULT;
1da177e4
LT		 3226
3227 if (len < 0)
3228 return -EINVAL;
1ce4f28b 3229
69e3c75f 3230 switch (optname) {
1da177e4 3231 case PACKET_STATISTICS:
1da177e4 3232 spin_lock_bh(&sk->sk_receive_queue.lock);
f6fb8f10 3233 if (po->tp_version == TPACKET_V3) {
c06fff6e 3234 lv = sizeof(struct tpacket_stats_v3);
f6fb8f10 3235 memcpy(&st_u.stats3, &po->stats,
c06fff6e 3236 sizeof(struct tpacket_stats));
f6fb8f10 3237 st_u.stats3.tp_freeze_q_cnt =
c06fff6e 3238 po->stats_u.stats3.tp_freeze_q_cnt;
f6fb8f10 3239 st_u.stats3.tp_packets += po->stats.tp_drops;
3240 data = &st_u.stats3;
3241 } else {
c06fff6e 3242 lv = sizeof(struct tpacket_stats);
f6fb8f10 3243 st = po->stats;
3244 st.tp_packets += st.tp_drops;
3245 data = &st;
3246 }
1da177e4
LT		 3247 memset(&po->stats, 0, sizeof(st));
3248 spin_unlock_bh(&sk->sk_receive_queue.lock);
8dc41944
HX		 3249 break;
3250 case PACKET_AUXDATA:
8dc41944 3251 val = po->auxdata;
80feaacb
PWJ		 3252 break;
3253 case PACKET_ORIGDEV:
80feaacb 3254 val = po->origdev;
bfd5f4a3
SS		 3255 break;
3256 case PACKET_VNET_HDR:
bfd5f4a3 3257 val = po->has_vnet_hdr;
1da177e4 3258 break;
bbd6ef87 3259 case PACKET_VERSION:
bbd6ef87 3260 val = po->tp_version;
bbd6ef87
PM		 3261 break;
3262 case PACKET_HDRLEN:
3263 if (len > sizeof(int))
3264 len = sizeof(int);
3265 if (copy_from_user(&val, optval, len))
3266 return -EFAULT;
3267 switch (val) {
3268 case TPACKET_V1:
3269 val = sizeof(struct tpacket_hdr);
3270 break;
3271 case TPACKET_V2:
3272 val = sizeof(struct tpacket2_hdr);
3273 break;
f6fb8f10 3274 case TPACKET_V3:
3275 val = sizeof(struct tpacket3_hdr);
3276 break;
bbd6ef87
PM		 3277 default:
3278 return -EINVAL;
3279 }
bbd6ef87 3280 break;
8913336a 3281 case PACKET_RESERVE:
8913336a 3282 val = po->tp_reserve;
8913336a 3283 break;
69e3c75f 3284 case PACKET_LOSS:
69e3c75f 3285 val = po->tp_loss;
69e3c75f 3286 break;
614f60fa 3287 case PACKET_TIMESTAMP:
614f60fa 3288 val = po->tp_tstamp;
614f60fa 3289 break;
dc99f600 3290 case PACKET_FANOUT:
dc99f600
DM		 3291 val = (po->fanout ?
3292 ((u32)po->fanout->id |
77f65ebd
WB		 3293 ((u32)po->fanout->type << 16) |
3294 ((u32)po->fanout->flags << 24)) :
dc99f600 3295 0);
dc99f600 3296 break;
5920cd3a
PC		 3297 case PACKET_TX_HAS_OFF:
3298 val = po->tp_tx_has_off;
3299 break;
1da177e4
LT		 3300 default:
3301 return -ENOPROTOOPT;
3302 }
3303
c06fff6e
ED		 3304 if (len > lv)
3305 len = lv;
8ae55f04
KK		 3306 if (put_user(len, optlen))
3307 return -EFAULT;
8dc41944
HX		 3308 if (copy_to_user(optval, data, len))
3309 return -EFAULT;
8ae55f04 3310 return 0;
1da177e4
LT		 3311}
3312
3313
3314static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
3315{
3316 struct sock *sk;
ad930650 3317 struct net_device *dev = data;
c346dca1 3318 struct net *net = dev_net(dev);
1da177e4 3319
808f5114 3320 rcu_read_lock();
b67bfe0d 3321 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3322 struct packet_sock *po = pkt_sk(sk);
3323
3324 switch (msg) {
3325 case NETDEV_UNREGISTER:
1da177e4
LT		 3326 if (po->mclist)
3327 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3328 /* fallthrough */
3329
1da177e4
LT		 3330 case NETDEV_DOWN:
3331 if (dev->ifindex == po->ifindex) {
3332 spin_lock(&po->bind_lock);
3333 if (po->running) {
ce06b03e 3334 __unregister_prot_hook(sk, false);
1da177e4
LT		 3335 sk->sk_err = ENETDOWN;
3336 if (!sock_flag(sk, SOCK_DEAD))
3337 sk->sk_error_report(sk);
3338 }
3339 if (msg == NETDEV_UNREGISTER) {
3340 po->ifindex = -1;
160ff18a
BG		 3341 if (po->prot_hook.dev)
3342 dev_put(po->prot_hook.dev);
1da177e4
LT		 3343 po->prot_hook.dev = NULL;
3344 }
3345 spin_unlock(&po->bind_lock);
3346 }
3347 break;
3348 case NETDEV_UP:
808f5114 3349 if (dev->ifindex == po->ifindex) {
3350 spin_lock(&po->bind_lock);
ce06b03e
DM		 3351 if (po->num)
3352 register_prot_hook(sk);
808f5114 3353 spin_unlock(&po->bind_lock);
1da177e4 3354 }
1da177e4
LT		 3355 break;
3356 }
3357 }
808f5114 3358 rcu_read_unlock();
1da177e4
LT		 3359 return NOTIFY_DONE;
3360}
3361
3362
3363static int packet_ioctl(struct socket *sock, unsigned int cmd,
3364 unsigned long arg)
3365{
3366 struct sock *sk = sock->sk;
3367
69e3c75f 3368 switch (cmd) {
40d4e3df
ED		 3369 case SIOCOUTQ:
3370 {
3371 int amount = sk_wmem_alloc_get(sk);
31e6d363 3372
40d4e3df
ED		 3373 return put_user(amount, (int __user *)arg);
3374 }
3375 case SIOCINQ:
3376 {
3377 struct sk_buff *skb;
3378 int amount = 0;
3379
3380 spin_lock_bh(&sk->sk_receive_queue.lock);
3381 skb = skb_peek(&sk->sk_receive_queue);
3382 if (skb)
3383 amount = skb->len;
3384 spin_unlock_bh(&sk->sk_receive_queue.lock);
3385 return put_user(amount, (int __user *)arg);
3386 }
3387 case SIOCGSTAMP:
3388 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3389 case SIOCGSTAMPNS:
3390 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3391
1da177e4 3392#ifdef CONFIG_INET
40d4e3df
ED		 3393 case SIOCADDRT:
3394 case SIOCDELRT:
3395 case SIOCDARP:
3396 case SIOCGARP:
3397 case SIOCSARP:
3398 case SIOCGIFADDR:
3399 case SIOCSIFADDR:
3400 case SIOCGIFBRDADDR:
3401 case SIOCSIFBRDADDR:
3402 case SIOCGIFNETMASK:
3403 case SIOCSIFNETMASK:
3404 case SIOCGIFDSTADDR:
3405 case SIOCSIFDSTADDR:
3406 case SIOCSIFFLAGS:
40d4e3df 3407 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3408#endif
3409
40d4e3df
ED		 3410 default:
3411 return -ENOIOCTLCMD;
1da177e4
LT		 3412 }
3413 return 0;
3414}
3415
40d4e3df 3416static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3417 poll_table *wait)
3418{
3419 struct sock *sk = sock->sk;
3420 struct packet_sock *po = pkt_sk(sk);
3421 unsigned int mask = datagram_poll(file, sock, wait);
3422
3423 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3424 if (po->rx_ring.pg_vec) {
f6fb8f10 3425 if (!packet_previous_rx_frame(po, &po->rx_ring,
3426 TP_STATUS_KERNEL))
1da177e4
LT		 3427 mask |= POLLIN | POLLRDNORM;
3428 }
3429 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3430 spin_lock_bh(&sk->sk_write_queue.lock);
3431 if (po->tx_ring.pg_vec) {
3432 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3433 mask |= POLLOUT | POLLWRNORM;
3434 }
3435 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3436 return mask;
3437}
3438
3439
3440/* Dirty? Well, I still did not learn better way to account
3441 * for user mmaps.
3442 */
3443
3444static void packet_mm_open(struct vm_area_struct *vma)
3445{
3446 struct file *file = vma->vm_file;
40d4e3df 3447 struct socket *sock = file->private_data;
1da177e4 3448 struct sock *sk = sock->sk;
1ce4f28b 3449
1da177e4
LT		 3450 if (sk)
3451 atomic_inc(&pkt_sk(sk)->mapped);
3452}
3453
3454static void packet_mm_close(struct vm_area_struct *vma)
3455{
3456 struct file *file = vma->vm_file;
40d4e3df 3457 struct socket *sock = file->private_data;
1da177e4 3458 struct sock *sk = sock->sk;
1ce4f28b 3459
1da177e4
LT		 3460 if (sk)
3461 atomic_dec(&pkt_sk(sk)->mapped);
3462}
3463
f0f37e2f 3464static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3465 .open = packet_mm_open,
3466 .close = packet_mm_close,
1da177e4
LT		 3467};
3468
0e3125c7
NH		 3469static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3470 unsigned int len)
1da177e4
LT		 3471{
3472 int i;
3473
4ebf0ae2 3474 for (i = 0; i < len; i++) {
0e3125c7 3475 if (likely(pg_vec[i].buffer)) {
c56b4d90 3476 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3477 vfree(pg_vec[i].buffer);
3478 else
3479 free_pages((unsigned long)pg_vec[i].buffer,
3480 order);
3481 pg_vec[i].buffer = NULL;
3482 }
1da177e4
LT		 3483 }
3484 kfree(pg_vec);
3485}
3486
eea49cc9 3487static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3488{
0e3125c7
NH		 3489 char *buffer = NULL;
3490 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3491 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3492
3493 buffer = (char *) __get_free_pages(gfp_flags, order);
3494
3495 if (buffer)
3496 return buffer;
3497
3498 /*
3499 * __get_free_pages failed, fall back to vmalloc
3500 */
bbce5a59 3501 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3502
0e3125c7
NH		 3503 if (buffer)
3504 return buffer;
3505
3506 /*
3507 * vmalloc failed, lets dig into swap here
3508 */
0e3125c7
NH		 3509 gfp_flags &= ~__GFP_NORETRY;
3510 buffer = (char *)__get_free_pages(gfp_flags, order);
3511 if (buffer)
3512 return buffer;
3513
3514 /*
3515 * complete and utter failure
3516 */
3517 return NULL;
4ebf0ae2
DM		 3518}
3519
0e3125c7 3520static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3521{
3522 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3523 struct pgv *pg_vec;
4ebf0ae2
DM		 3524 int i;
3525
0e3125c7 3526 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3527 if (unlikely(!pg_vec))
3528 goto out;
3529
3530 for (i = 0; i < block_nr; i++) {
c56b4d90 3531 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3532 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3533 goto out_free_pgvec;
3534 }
3535
3536out:
3537 return pg_vec;
3538
3539out_free_pgvec:
3540 free_pg_vec(pg_vec, order, block_nr);
3541 pg_vec = NULL;
3542 goto out;
3543}
1da177e4 3544
f6fb8f10 3545static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3546 int closing, int tx_ring)
1da177e4 3547{
0e3125c7 3548 struct pgv *pg_vec = NULL;
1da177e4 3549 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3550 int was_running, order = 0;
69e3c75f
JB		 3551 struct packet_ring_buffer *rb;
3552 struct sk_buff_head *rb_queue;
0e11c91e 3553 __be16 num;
f6fb8f10 3554 int err = -EINVAL;
3555 /* Added to avoid minimal code churn */
3556 struct tpacket_req *req = &req_u->req;
3557
3558 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3559 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3560 WARN(1, "Tx-ring is not supported.\n");
3561 goto out;
3562 }
1ce4f28b 3563
69e3c75f
JB		 3564 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3565 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3566
69e3c75f
JB		 3567 err = -EBUSY;
3568 if (!closing) {
3569 if (atomic_read(&po->mapped))
3570 goto out;
3571 if (atomic_read(&rb->pending))
3572 goto out;
3573 }
1da177e4 3574
69e3c75f
JB		 3575 if (req->tp_block_nr) {
3576 /* Sanity tests and some calculations */
3577 err = -EBUSY;
3578 if (unlikely(rb->pg_vec))
3579 goto out;
1da177e4 3580
bbd6ef87
PM		 3581 switch (po->tp_version) {
3582 case TPACKET_V1:
3583 po->tp_hdrlen = TPACKET_HDRLEN;
3584 break;
3585 case TPACKET_V2:
3586 po->tp_hdrlen = TPACKET2_HDRLEN;
3587 break;
f6fb8f10 3588 case TPACKET_V3:
3589 po->tp_hdrlen = TPACKET3_HDRLEN;
3590 break;
bbd6ef87
PM		 3591 }
3592
69e3c75f 3593 err = -EINVAL;
4ebf0ae2 3594 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3595 goto out;
4ebf0ae2 3596 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3597 goto out;
8913336a 3598 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3599 po->tp_reserve))
3600 goto out;
4ebf0ae2 3601 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3602 goto out;
1da177e4 3603
69e3c75f
JB		 3604 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3605 if (unlikely(rb->frames_per_block <= 0))
3606 goto out;
3607 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3608 req->tp_frame_nr))
3609 goto out;
1da177e4
LT		 3610
3611 err = -ENOMEM;
4ebf0ae2
DM		 3612 order = get_order(req->tp_block_size);
3613 pg_vec = alloc_pg_vec(req, order);
3614 if (unlikely(!pg_vec))
1da177e4 3615 goto out;
f6fb8f10 3616 switch (po->tp_version) {
3617 case TPACKET_V3:
3618 /* Transmit path is not supported. We checked
3619 * it above but just being paranoid
3620 */
3621 if (!tx_ring)
3622 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3623 break;
3624 default:
3625 break;
3626 }
69e3c75f
JB		 3627 }
3628 /* Done */
3629 else {
3630 err = -EINVAL;
4ebf0ae2 3631 if (unlikely(req->tp_frame_nr))
69e3c75f 3632 goto out;
1da177e4
LT		 3633 }
3634
3635 lock_sock(sk);
3636
3637 /* Detach socket from network */
3638 spin_lock(&po->bind_lock);
3639 was_running = po->running;
3640 num = po->num;
3641 if (was_running) {
1da177e4 3642 po->num = 0;
ce06b03e 3643 __unregister_prot_hook(sk, false);
1da177e4
LT		 3644 }
3645 spin_unlock(&po->bind_lock);
1ce4f28b 3646
1da177e4
LT		 3647 synchronize_net();
3648
3649 err = -EBUSY;
905db440 3650 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3651 if (closing || atomic_read(&po->mapped) == 0) {
3652 err = 0;
69e3c75f 3653 spin_lock_bh(&rb_queue->lock);
c053fd96 3654 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3655 rb->frame_max = (req->tp_frame_nr - 1);
3656 rb->head = 0;
3657 rb->frame_size = req->tp_frame_size;
3658 spin_unlock_bh(&rb_queue->lock);
3659
c053fd96
CG		 3660 swap(rb->pg_vec_order, order);
3661 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3662
3663 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3664 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3665 tpacket_rcv : packet_rcv;
3666 skb_queue_purge(rb_queue);
1da177e4 3667 if (atomic_read(&po->mapped))
40d4e3df
ED		 3668 pr_err("packet_mmap: vma is busy: %d\n",
3669 atomic_read(&po->mapped));
1da177e4 3670 }
905db440 3671 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3672
3673 spin_lock(&po->bind_lock);
ce06b03e 3674 if (was_running) {
1da177e4 3675 po->num = num;
ce06b03e 3676 register_prot_hook(sk);
1da177e4
LT		 3677 }
3678 spin_unlock(&po->bind_lock);
f6fb8f10 3679 if (closing && (po->tp_version > TPACKET_V2)) {
3680 /* Because we don't support block-based V3 on tx-ring */
3681 if (!tx_ring)
3682 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3683 }
1da177e4
LT		 3684 release_sock(sk);
3685
1da177e4
LT		 3686 if (pg_vec)
3687 free_pg_vec(pg_vec, order, req->tp_block_nr);
3688out:
3689 return err;
3690}
3691
69e3c75f
JB		 3692static int packet_mmap(struct file *file, struct socket *sock,
3693 struct vm_area_struct *vma)
1da177e4
LT		 3694{
3695 struct sock *sk = sock->sk;
3696 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3697 unsigned long size, expected_size;
3698 struct packet_ring_buffer *rb;
1da177e4
LT		 3699 unsigned long start;
3700 int err = -EINVAL;
3701 int i;
3702
3703 if (vma->vm_pgoff)
3704 return -EINVAL;
3705
905db440 3706 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3707
3708 expected_size = 0;
3709 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3710 if (rb->pg_vec) {
3711 expected_size += rb->pg_vec_len
3712 * rb->pg_vec_pages
3713 * PAGE_SIZE;
3714 }
3715 }
3716
3717 if (expected_size == 0)
1da177e4 3718 goto out;
69e3c75f
JB		 3719
3720 size = vma->vm_end - vma->vm_start;
3721 if (size != expected_size)
1da177e4
LT		 3722 goto out;
3723
1da177e4 3724 start = vma->vm_start;
69e3c75f
JB		 3725 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3726 if (rb->pg_vec == NULL)
3727 continue;
3728
3729 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3730 struct page *page;
3731 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3732 int pg_num;
3733
c56b4d90
CG		 3734 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3735 page = pgv_to_page(kaddr);
69e3c75f
JB		 3736 err = vm_insert_page(vma, start, page);
3737 if (unlikely(err))
3738 goto out;
3739 start += PAGE_SIZE;
0e3125c7 3740 kaddr += PAGE_SIZE;
69e3c75f 3741 }
4ebf0ae2 3742 }
1da177e4 3743 }
69e3c75f 3744
4ebf0ae2 3745 atomic_inc(&po->mapped);
1da177e4
LT		 3746 vma->vm_ops = &packet_mmap_ops;
3747 err = 0;
3748
3749out:
905db440 3750 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3751 return err;
3752}
1da177e4 3753
90ddc4f0 3754static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3755 .family = PF_PACKET,
3756 .owner = THIS_MODULE,
3757 .release = packet_release,
3758 .bind = packet_bind_spkt,
3759 .connect = sock_no_connect,
3760 .socketpair = sock_no_socketpair,
3761 .accept = sock_no_accept,
3762 .getname = packet_getname_spkt,
3763 .poll = datagram_poll,
3764 .ioctl = packet_ioctl,
3765 .listen = sock_no_listen,
3766 .shutdown = sock_no_shutdown,
3767 .setsockopt = sock_no_setsockopt,
3768 .getsockopt = sock_no_getsockopt,
3769 .sendmsg = packet_sendmsg_spkt,
3770 .recvmsg = packet_recvmsg,
3771 .mmap = sock_no_mmap,
3772 .sendpage = sock_no_sendpage,
3773};
1da177e4 3774
90ddc4f0 3775static const struct proto_ops packet_ops = {
1da177e4
LT		 3776 .family = PF_PACKET,
3777 .owner = THIS_MODULE,
3778 .release = packet_release,
3779 .bind = packet_bind,
3780 .connect = sock_no_connect,
3781 .socketpair = sock_no_socketpair,
3782 .accept = sock_no_accept,
1ce4f28b 3783 .getname = packet_getname,
1da177e4
LT		 3784 .poll = packet_poll,
3785 .ioctl = packet_ioctl,
3786 .listen = sock_no_listen,
3787 .shutdown = sock_no_shutdown,
3788 .setsockopt = packet_setsockopt,
3789 .getsockopt = packet_getsockopt,
3790 .sendmsg = packet_sendmsg,
3791 .recvmsg = packet_recvmsg,
3792 .mmap = packet_mmap,
3793 .sendpage = sock_no_sendpage,
3794};
3795
ec1b4cf7 3796static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3797 .family = PF_PACKET,
3798 .create = packet_create,
3799 .owner = THIS_MODULE,
3800};
3801
3802static struct notifier_block packet_netdev_notifier = {
40d4e3df 3803 .notifier_call = packet_notifier,
1da177e4
LT		 3804};
3805
3806#ifdef CONFIG_PROC_FS
1da177e4
LT		 3807
3808static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3809 __acquires(RCU)
1da177e4 3810{
e372c414 3811 struct net *net = seq_file_net(seq);
808f5114 3812
3813 rcu_read_lock();
3814 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3815}
3816
3817static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3818{
1bf40954 3819 struct net *net = seq_file_net(seq);
808f5114 3820 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3821}
3822
3823static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3824 __releases(RCU)
1da177e4 3825{
808f5114 3826 rcu_read_unlock();
1da177e4
LT		 3827}
3828
1ce4f28b 3829static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3830{
3831 if (v == SEQ_START_TOKEN)
3832 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3833 else {
b7ceabd9 3834 struct sock *s = sk_entry(v);
1da177e4
LT		 3835 const struct packet_sock *po = pkt_sk(s);
3836
3837 seq_printf(seq,
71338aa7 3838 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3839 s,
3840 atomic_read(&s->sk_refcnt),
3841 s->sk_type,
3842 ntohs(po->num),
3843 po->ifindex,
3844 po->running,
3845 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 3846 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 3847 sock_i_ino(s));
1da177e4
LT		 3848 }
3849
3850 return 0;
3851}
3852
56b3d975 3853static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3854 .start = packet_seq_start,
3855 .next = packet_seq_next,
3856 .stop = packet_seq_stop,
3857 .show = packet_seq_show,
3858};
3859
3860static int packet_seq_open(struct inode *inode, struct file *file)
3861{
e372c414
DL		 3862 return seq_open_net(inode, file, &packet_seq_ops,
3863 sizeof(struct seq_net_private));
1da177e4
LT		 3864}
3865
da7071d7 3866static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3867 .owner = THIS_MODULE,
3868 .open = packet_seq_open,
3869 .read = seq_read,
3870 .llseek = seq_lseek,
e372c414 3871 .release = seq_release_net,
1da177e4
LT		 3872};
3873
3874#endif
3875
2c8c1e72 3876static int __net_init packet_net_init(struct net *net)
d12d01d6 3877{
0fa7fa98 3878 mutex_init(&net->packet.sklist_lock);
2aaef4e4 3879 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 3880
d4beaa66 3881 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 3882 return -ENOMEM;
3883
3884 return 0;
3885}
3886
2c8c1e72 3887static void __net_exit packet_net_exit(struct net *net)
d12d01d6 3888{
ece31ffd 3889 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 3890}
3891
3892static struct pernet_operations packet_net_ops = {
3893 .init = packet_net_init,
3894 .exit = packet_net_exit,
3895};
3896
3897
1da177e4
LT		 3898static void __exit packet_exit(void)
3899{
1da177e4 3900 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3901 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3902 sock_unregister(PF_PACKET);
3903 proto_unregister(&packet_proto);
3904}
3905
3906static int __init packet_init(void)
3907{
3908 int rc = proto_register(&packet_proto, 0);
3909
3910 if (rc != 0)
3911 goto out;
3912
3913 sock_register(&packet_family_ops);
d12d01d6 3914 register_pernet_subsys(&packet_net_ops);
1da177e4 3915 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3916out:
3917 return rc;
3918}
3919
3920module_init(packet_init);
3921module_exit(packet_exit);
3922MODULE_LICENSE("GPL");
3923MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 1.311505 seconds and 5 git commands to generate.