projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bcma: only map wrapper if its address is available
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
b0138408 91#include <linux/percpu.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
d346a3fa
DB		 240static int packet_direct_xmit(struct sk_buff *skb)
241{
242 struct net_device *dev = skb->dev;
243 const struct net_device_ops *ops = dev->netdev_ops;
244 netdev_features_t features;
245 struct netdev_queue *txq;
43279500 246 int ret = NETDEV_TX_BUSY;
d346a3fa 247 u16 queue_map;
d346a3fa
DB		 248
249 if (unlikely(!netif_running(dev) ||
43279500
DB		 250 !netif_carrier_ok(dev)))
251 goto drop;
d346a3fa
DB		 252
253 features = netif_skb_features(skb);
254 if (skb_needs_linearize(skb, features) &&
43279500
DB		 255 __skb_linearize(skb))
256 goto drop;
d346a3fa
DB		 257
258 queue_map = skb_get_queue_mapping(skb);
259 txq = netdev_get_tx_queue(dev, queue_map);
260
43279500
DB		 261 local_bh_disable();
262
263 HARD_TX_LOCK(dev, txq, smp_processor_id());
8e2f1a63 264 if (!netif_xmit_frozen_or_drv_stopped(txq)) {
43279500
DB		 265 ret = ops->ndo_start_xmit(skb, dev);
266 if (ret == NETDEV_TX_OK)
267 txq_trans_update(txq);
d346a3fa 268 }
43279500 269 HARD_TX_UNLOCK(dev, txq);
d346a3fa 270
43279500
DB		 271 local_bh_enable();
272
273 if (!dev_xmit_complete(ret))
d346a3fa 274 kfree_skb(skb);
43279500 275
d346a3fa 276 return ret;
43279500 277drop:
0f97ede4 278 atomic_long_inc(&dev->tx_dropped);
43279500
DB		 279 kfree_skb(skb);
280 return NET_XMIT_DROP;
d346a3fa
DB		 281}
282
66e56cd4
DB		 283static struct net_device *packet_cached_dev_get(struct packet_sock *po)
284{
285 struct net_device *dev;
286
287 rcu_read_lock();
288 dev = rcu_dereference(po->cached_dev);
289 if (likely(dev))
290 dev_hold(dev);
291 rcu_read_unlock();
292
293 return dev;
294}
295
296static void packet_cached_dev_assign(struct packet_sock *po,
297 struct net_device *dev)
298{
299 rcu_assign_pointer(po->cached_dev, dev);
300}
301
302static void packet_cached_dev_reset(struct packet_sock *po)
303{
304 RCU_INIT_POINTER(po->cached_dev, NULL);
305}
306
d346a3fa
DB		 307static bool packet_use_direct_xmit(const struct packet_sock *po)
308{
309 return po->xmit == packet_direct_xmit;
310}
311
0fd5d57b 312static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
d346a3fa 313{
1cbac010 314 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 315}
316
0fd5d57b
DB		 317static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
318{
319 const struct net_device_ops *ops = dev->netdev_ops;
320 u16 queue_index;
321
322 if (ops->ndo_select_queue) {
323 queue_index = ops->ndo_select_queue(dev, skb, NULL,
324 __packet_pick_tx_queue);
325 queue_index = netdev_cap_txqueue(dev, queue_index);
326 } else {
327 queue_index = __packet_pick_tx_queue(dev, skb);
328 }
329
330 skb_set_queue_mapping(skb, queue_index);
331}
332
ce06b03e
DM		 333/* register_prot_hook must be invoked with the po->bind_lock held,
334 * or from a context in which asynchronous accesses to the packet
335 * socket is not possible (packet_create()).
336 */
337static void register_prot_hook(struct sock *sk)
338{
339 struct packet_sock *po = pkt_sk(sk);
e40526cb 340
ce06b03e 341 if (!po->running) {
66e56cd4 342 if (po->fanout)
dc99f600 343 __fanout_link(sk, po);
66e56cd4 344 else
dc99f600 345 dev_add_pack(&po->prot_hook);
e40526cb 346
ce06b03e
DM		 347 sock_hold(sk);
348 po->running = 1;
349 }
350}
351
352/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
353 * held. If the sync parameter is true, we will temporarily drop
354 * the po->bind_lock and do a synchronize_net to make sure no
355 * asynchronous packet processing paths still refer to the elements
356 * of po->prot_hook. If the sync parameter is false, it is the
357 * callers responsibility to take care of this.
358 */
359static void __unregister_prot_hook(struct sock *sk, bool sync)
360{
361 struct packet_sock *po = pkt_sk(sk);
362
363 po->running = 0;
66e56cd4
DB		 364
365 if (po->fanout)
dc99f600 366 __fanout_unlink(sk, po);
66e56cd4 367 else
dc99f600 368 __dev_remove_pack(&po->prot_hook);
e40526cb 369
ce06b03e
DM		 370 __sock_put(sk);
371
372 if (sync) {
373 spin_unlock(&po->bind_lock);
374 synchronize_net();
375 spin_lock(&po->bind_lock);
376 }
377}
378
379static void unregister_prot_hook(struct sock *sk, bool sync)
380{
381 struct packet_sock *po = pkt_sk(sk);
382
383 if (po->running)
384 __unregister_prot_hook(sk, sync);
385}
386
f6dafa95 387static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 388{
389 if (is_vmalloc_addr(addr))
390 return vmalloc_to_page(addr);
391 return virt_to_page(addr);
392}
393
69e3c75f 394static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 395{
184f489e 396 union tpacket_uhdr h;
1da177e4 397
69e3c75f 398 h.raw = frame;
bbd6ef87
PM		 399 switch (po->tp_version) {
400 case TPACKET_V1:
69e3c75f 401 h.h1->tp_status = status;
0af55bb5 402 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 403 break;
404 case TPACKET_V2:
69e3c75f 405 h.h2->tp_status = status;
0af55bb5 406 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 407 break;
f6fb8f10 408 case TPACKET_V3:
69e3c75f 409 default:
f6fb8f10 410 WARN(1, "TPACKET version not supported.\n");
69e3c75f 411 BUG();
bbd6ef87 412 }
69e3c75f
JB		 413
414 smp_wmb();
bbd6ef87
PM		 415}
416
69e3c75f 417static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 418{
184f489e 419 union tpacket_uhdr h;
bbd6ef87 420
69e3c75f
JB		 421 smp_rmb();
422
bbd6ef87
PM		 423 h.raw = frame;
424 switch (po->tp_version) {
425 case TPACKET_V1:
0af55bb5 426 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 427 return h.h1->tp_status;
bbd6ef87 428 case TPACKET_V2:
0af55bb5 429 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 430 return h.h2->tp_status;
f6fb8f10 431 case TPACKET_V3:
69e3c75f 432 default:
f6fb8f10 433 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 434 BUG();
435 return 0;
bbd6ef87 436 }
1da177e4 437}
69e3c75f 438
b9c32fb2
DB		 439static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
440 unsigned int flags)
7a51384c
DB		 441{
442 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
443
68a360e8
WB		 444 if (shhwtstamps &&
445 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
446 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
447 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 448
449 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 450 return TP_STATUS_TS_SOFTWARE;
7a51384c 451
b9c32fb2 452 return 0;
7a51384c
DB		 453}
454
b9c32fb2
DB		 455static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
456 struct sk_buff *skb)
2e31396f
WB		 457{
458 union tpacket_uhdr h;
459 struct timespec ts;
b9c32fb2 460 __u32 ts_status;
2e31396f 461
b9c32fb2
DB		 462 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
463 return 0;
2e31396f
WB		 464
465 h.raw = frame;
466 switch (po->tp_version) {
467 case TPACKET_V1:
468 h.h1->tp_sec = ts.tv_sec;
469 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
470 break;
471 case TPACKET_V2:
472 h.h2->tp_sec = ts.tv_sec;
473 h.h2->tp_nsec = ts.tv_nsec;
474 break;
475 case TPACKET_V3:
476 default:
477 WARN(1, "TPACKET version not supported.\n");
478 BUG();
479 }
480
481 /* one flush is safe, as both fields always lie on the same cacheline */
482 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
483 smp_wmb();
b9c32fb2
DB		 484
485 return ts_status;
2e31396f
WB		 486}
487
69e3c75f
JB		 488static void *packet_lookup_frame(struct packet_sock *po,
489 struct packet_ring_buffer *rb,
490 unsigned int position,
491 int status)
492{
493 unsigned int pg_vec_pos, frame_offset;
184f489e 494 union tpacket_uhdr h;
69e3c75f
JB		 495
496 pg_vec_pos = position / rb->frames_per_block;
497 frame_offset = position % rb->frames_per_block;
498
0e3125c7
NH		 499 h.raw = rb->pg_vec[pg_vec_pos].buffer +
500 (frame_offset * rb->frame_size);
69e3c75f
JB		 501
502 if (status != __packet_get_status(po, h.raw))
503 return NULL;
504
505 return h.raw;
506}
507
eea49cc9 508static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 509 struct packet_ring_buffer *rb,
510 int status)
511{
512 return packet_lookup_frame(po, rb, rb->head, status);
513}
514
bc59ba39 515static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 516{
517 del_timer_sync(&pkc->retire_blk_timer);
518}
519
520static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
521 int tx_ring,
522 struct sk_buff_head *rb_queue)
523{
bc59ba39 524 struct tpacket_kbdq_core *pkc;
f6fb8f10 525
22781a5b
DJ		 526 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
527 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 528
ec6f809f 529 spin_lock_bh(&rb_queue->lock);
f6fb8f10 530 pkc->delete_blk_timer = 1;
ec6f809f 531 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 532
533 prb_del_retire_blk_timer(pkc);
534}
535
536static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 537 struct tpacket_kbdq_core *pkc,
f6fb8f10 538 void (*func) (unsigned long))
539{
540 init_timer(&pkc->retire_blk_timer);
541 pkc->retire_blk_timer.data = (long)po;
542 pkc->retire_blk_timer.function = func;
543 pkc->retire_blk_timer.expires = jiffies;
544}
545
546static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
547{
bc59ba39 548 struct tpacket_kbdq_core *pkc;
f6fb8f10 549
550 if (tx_ring)
551 BUG();
552
22781a5b
DJ		 553 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
554 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 555 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
556}
557
558static int prb_calc_retire_blk_tmo(struct packet_sock *po,
559 int blk_size_in_bytes)
560{
561 struct net_device *dev;
562 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 563 struct ethtool_cmd ecmd;
564 int err;
e440cf2c 565 u32 speed;
f6fb8f10 566
4bc71cb9
JP		 567 rtnl_lock();
568 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
569 if (unlikely(!dev)) {
570 rtnl_unlock();
f6fb8f10 571 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 572 }
573 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 574 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 575 rtnl_unlock();
576 if (!err) {
4bc71cb9
JP		 577 /*
578 * If the link speed is so slow you don't really
579 * need to worry about perf anyways
580 */
e440cf2c 581 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 582 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 583 } else {
584 msec = 1;
585 div = speed / 1000;
f6fb8f10 586 }
587 }
588
589 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
590
591 if (div)
592 mbits /= div;
593
594 tmo = mbits * msec;
595
596 if (div)
597 return tmo+1;
598 return tmo;
599}
600
bc59ba39 601static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 602 union tpacket_req_u *req_u)
603{
604 p1->feature_req_word = req_u->req3.tp_feature_req_word;
605}
606
607static void init_prb_bdqc(struct packet_sock *po,
608 struct packet_ring_buffer *rb,
609 struct pgv *pg_vec,
610 union tpacket_req_u *req_u, int tx_ring)
611{
22781a5b 612 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 613 struct tpacket_block_desc *pbd;
f6fb8f10 614
615 memset(p1, 0x0, sizeof(*p1));
616
617 p1->knxt_seq_num = 1;
618 p1->pkbdq = pg_vec;
bc59ba39 619 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 620 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 621 p1->kblk_size = req_u->req3.tp_block_size;
622 p1->knum_blocks = req_u->req3.tp_block_nr;
623 p1->hdrlen = po->tp_hdrlen;
624 p1->version = po->tp_version;
625 p1->last_kactive_blk_num = 0;
ee80fbf3 626 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 627 if (req_u->req3.tp_retire_blk_tov)
628 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
629 else
630 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
631 req_u->req3.tp_block_size);
632 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
633 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
634
dc808110 635 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
f6fb8f10 636 prb_init_ft_ops(p1, req_u);
637 prb_setup_retire_blk_timer(po, tx_ring);
638 prb_open_block(p1, pbd);
639}
640
641/* Do NOT update the last_blk_num first.
642 * Assumes sk_buff_head lock is held.
643 */
bc59ba39 644static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 645{
646 mod_timer(&pkc->retire_blk_timer,
647 jiffies + pkc->tov_in_jiffies);
648 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
649}
650
651/*
652 * Timer logic:
653 * 1) We refresh the timer only when we open a block.
654 * By doing this we don't waste cycles refreshing the timer
655 * on packet-by-packet basis.
656 *
657 * With a 1MB block-size, on a 1Gbps line, it will take
658 * i) ~8 ms to fill a block + ii) memcpy etc.
659 * In this cut we are not accounting for the memcpy time.
660 *
661 * So, if the user sets the 'tmo' to 10ms then the timer
662 * will never fire while the block is still getting filled
663 * (which is what we want). However, the user could choose
664 * to close a block early and that's fine.
665 *
666 * But when the timer does fire, we check whether or not to refresh it.
667 * Since the tmo granularity is in msecs, it is not too expensive
668 * to refresh the timer, lets say every '8' msecs.
669 * Either the user can set the 'tmo' or we can derive it based on
670 * a) line-speed and b) block-size.
671 * prb_calc_retire_blk_tmo() calculates the tmo.
672 *
673 */
674static void prb_retire_rx_blk_timer_expired(unsigned long data)
675{
676 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 677 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 678 unsigned int frozen;
bc59ba39 679 struct tpacket_block_desc *pbd;
f6fb8f10 680
681 spin_lock(&po->sk.sk_receive_queue.lock);
682
683 frozen = prb_queue_frozen(pkc);
684 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
685
686 if (unlikely(pkc->delete_blk_timer))
687 goto out;
688
689 /* We only need to plug the race when the block is partially filled.
690 * tpacket_rcv:
691 * lock(); increment BLOCK_NUM_PKTS; unlock()
692 * copy_bits() is in progress ...
693 * timer fires on other cpu:
694 * we can't retire the current block because copy_bits
695 * is in progress.
696 *
697 */
698 if (BLOCK_NUM_PKTS(pbd)) {
699 while (atomic_read(&pkc->blk_fill_in_prog)) {
700 /* Waiting for skb_copy_bits to finish... */
701 cpu_relax();
702 }
703 }
704
705 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
706 if (!frozen) {
707 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
708 if (!prb_dispatch_next_block(pkc, po))
709 goto refresh_timer;
710 else
711 goto out;
712 } else {
713 /* Case 1. Queue was frozen because user-space was
714 * lagging behind.
715 */
716 if (prb_curr_blk_in_use(pkc, pbd)) {
717 /*
718 * Ok, user-space is still behind.
719 * So just refresh the timer.
720 */
721 goto refresh_timer;
722 } else {
723 /* Case 2. queue was frozen,user-space caught up,
724 * now the link went idle && the timer fired.
725 * We don't have a block to close.So we open this
726 * block and restart the timer.
727 * opening a block thaws the queue,restarts timer
728 * Thawing/timer-refresh is a side effect.
729 */
730 prb_open_block(pkc, pbd);
731 goto out;
732 }
733 }
734 }
735
736refresh_timer:
737 _prb_refresh_rx_retire_blk_timer(pkc);
738
739out:
740 spin_unlock(&po->sk.sk_receive_queue.lock);
741}
742
eea49cc9 743static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 744 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 745{
746 /* Flush everything minus the block header */
747
748#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
749 u8 *start, *end;
750
751 start = (u8 *)pbd1;
752
753 /* Skip the block header(we know header WILL fit in 4K) */
754 start += PAGE_SIZE;
755
756 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
757 for (; start < end; start += PAGE_SIZE)
758 flush_dcache_page(pgv_to_page(start));
759
760 smp_wmb();
761#endif
762
763 /* Now update the block status. */
764
765 BLOCK_STATUS(pbd1) = status;
766
767 /* Flush the block header */
768
769#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
770 start = (u8 *)pbd1;
771 flush_dcache_page(pgv_to_page(start));
772
773 smp_wmb();
774#endif
775}
776
777/*
778 * Side effect:
779 *
780 * 1) flush the block
781 * 2) Increment active_blk_num
782 *
783 * Note:We DONT refresh the timer on purpose.
784 * Because almost always the next block will be opened.
785 */
bc59ba39 786static void prb_close_block(struct tpacket_kbdq_core *pkc1,
787 struct tpacket_block_desc *pbd1,
f6fb8f10 788 struct packet_sock *po, unsigned int stat)
789{
790 __u32 status = TP_STATUS_USER | stat;
791
792 struct tpacket3_hdr *last_pkt;
bc59ba39 793 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 794
ee80fbf3 795 if (po->stats.stats3.tp_drops)
f6fb8f10 796 status |= TP_STATUS_LOSING;
797
798 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
799 last_pkt->tp_next_offset = 0;
800
801 /* Get the ts of the last pkt */
802 if (BLOCK_NUM_PKTS(pbd1)) {
803 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
804 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
805 } else {
806 /* Ok, we tmo'd - so get the current time */
807 struct timespec ts;
808 getnstimeofday(&ts);
809 h1->ts_last_pkt.ts_sec = ts.tv_sec;
810 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
811 }
812
813 smp_wmb();
814
815 /* Flush the block */
816 prb_flush_block(pkc1, pbd1, status);
817
818 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
819}
820
eea49cc9 821static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 822{
823 pkc->reset_pending_on_curr_blk = 0;
824}
825
826/*
827 * Side effect of opening a block:
828 *
829 * 1) prb_queue is thawed.
830 * 2) retire_blk_timer is refreshed.
831 *
832 */
bc59ba39 833static void prb_open_block(struct tpacket_kbdq_core *pkc1,
834 struct tpacket_block_desc *pbd1)
f6fb8f10 835{
836 struct timespec ts;
bc59ba39 837 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 838
839 smp_rmb();
840
8da3056c
DB		 841 /* We could have just memset this but we will lose the
842 * flexibility of making the priv area sticky
843 */
f6fb8f10 844
8da3056c
DB		 845 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
846 BLOCK_NUM_PKTS(pbd1) = 0;
847 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 848
8da3056c
DB		 849 getnstimeofday(&ts);
850
851 h1->ts_first_pkt.ts_sec = ts.tv_sec;
852 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 853
8da3056c
DB		 854 pkc1->pkblk_start = (char *)pbd1;
855 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
856
857 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
858 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
859
860 pbd1->version = pkc1->version;
861 pkc1->prev = pkc1->nxt_offset;
862 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
863
864 prb_thaw_queue(pkc1);
865 _prb_refresh_rx_retire_blk_timer(pkc1);
866
867 smp_wmb();
f6fb8f10 868}
869
870/*
871 * Queue freeze logic:
872 * 1) Assume tp_block_nr = 8 blocks.
873 * 2) At time 't0', user opens Rx ring.
874 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
875 * 4) user-space is either sleeping or processing block '0'.
876 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
877 * it will close block-7,loop around and try to fill block '0'.
878 * call-flow:
879 * __packet_lookup_frame_in_block
880 * prb_retire_current_block()
881 * prb_dispatch_next_block()
882 * |->(BLOCK_STATUS == USER) evaluates to true
883 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
884 * 6) Now there are two cases:
885 * 6.1) Link goes idle right after the queue is frozen.
886 * But remember, the last open_block() refreshed the timer.
887 * When this timer expires,it will refresh itself so that we can
888 * re-open block-0 in near future.
889 * 6.2) Link is busy and keeps on receiving packets. This is a simple
890 * case and __packet_lookup_frame_in_block will check if block-0
891 * is free and can now be re-used.
892 */
eea49cc9 893static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 894 struct packet_sock *po)
895{
896 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 897 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 898}
899
900#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
901
902/*
903 * If the next block is free then we will dispatch it
904 * and return a good offset.
905 * Else, we will freeze the queue.
906 * So, caller must check the return value.
907 */
bc59ba39 908static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 909 struct packet_sock *po)
910{
bc59ba39 911 struct tpacket_block_desc *pbd;
f6fb8f10 912
913 smp_rmb();
914
915 /* 1. Get current block num */
916 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
917
918 /* 2. If this block is currently in_use then freeze the queue */
919 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
920 prb_freeze_queue(pkc, po);
921 return NULL;
922 }
923
924 /*
925 * 3.
926 * open this block and return the offset where the first packet
927 * needs to get stored.
928 */
929 prb_open_block(pkc, pbd);
930 return (void *)pkc->nxt_offset;
931}
932
bc59ba39 933static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 934 struct packet_sock *po, unsigned int status)
935{
bc59ba39 936 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 937
938 /* retire/close the current block */
939 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
940 /*
941 * Plug the case where copy_bits() is in progress on
942 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
943 * have space to copy the pkt in the current block and
944 * called prb_retire_current_block()
945 *
946 * We don't need to worry about the TMO case because
947 * the timer-handler already handled this case.
948 */
949 if (!(status & TP_STATUS_BLK_TMO)) {
950 while (atomic_read(&pkc->blk_fill_in_prog)) {
951 /* Waiting for skb_copy_bits to finish... */
952 cpu_relax();
953 }
954 }
955 prb_close_block(pkc, pbd, po, status);
956 return;
957 }
f6fb8f10 958}
959
eea49cc9 960static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 961 struct tpacket_block_desc *pbd)
f6fb8f10 962{
963 return TP_STATUS_USER & BLOCK_STATUS(pbd);
964}
965
eea49cc9 966static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 967{
968 return pkc->reset_pending_on_curr_blk;
969}
970
eea49cc9 971static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 972{
bc59ba39 973 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 974 atomic_dec(&pkc->blk_fill_in_prog);
975}
976
eea49cc9 977static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 978 struct tpacket3_hdr *ppd)
979{
3958afa1 980 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 981}
982
eea49cc9 983static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 984 struct tpacket3_hdr *ppd)
985{
986 ppd->hv1.tp_rxhash = 0;
987}
988
eea49cc9 989static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 990 struct tpacket3_hdr *ppd)
991{
992 if (vlan_tx_tag_present(pkc->skb)) {
993 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
a0cdfcf3
AW		 994 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
995 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 996 } else {
9e67030a 997 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 998 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 999 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 1000 }
1001}
1002
bc59ba39 1003static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 1004 struct tpacket3_hdr *ppd)
1005{
a0cdfcf3 1006 ppd->hv1.tp_padding = 0;
f6fb8f10 1007 prb_fill_vlan_info(pkc, ppd);
1008
1009 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1010 prb_fill_rxhash(pkc, ppd);
1011 else
1012 prb_clear_rxhash(pkc, ppd);
1013}
1014
eea49cc9 1015static void prb_fill_curr_block(char *curr,
bc59ba39 1016 struct tpacket_kbdq_core *pkc,
1017 struct tpacket_block_desc *pbd,
f6fb8f10 1018 unsigned int len)
1019{
1020 struct tpacket3_hdr *ppd;
1021
1022 ppd = (struct tpacket3_hdr *)curr;
1023 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1024 pkc->prev = curr;
1025 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1026 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1027 BLOCK_NUM_PKTS(pbd) += 1;
1028 atomic_inc(&pkc->blk_fill_in_prog);
1029 prb_run_all_ft_ops(pkc, ppd);
1030}
1031
1032/* Assumes caller has the sk->rx_queue.lock */
1033static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1034 struct sk_buff *skb,
1035 int status,
1036 unsigned int len
1037 )
1038{
bc59ba39 1039 struct tpacket_kbdq_core *pkc;
1040 struct tpacket_block_desc *pbd;
f6fb8f10 1041 char *curr, *end;
1042
e3192690 1043 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1044 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1045
1046 /* Queue is frozen when user space is lagging behind */
1047 if (prb_queue_frozen(pkc)) {
1048 /*
1049 * Check if that last block which caused the queue to freeze,
1050 * is still in_use by user-space.
1051 */
1052 if (prb_curr_blk_in_use(pkc, pbd)) {
1053 /* Can't record this packet */
1054 return NULL;
1055 } else {
1056 /*
1057 * Ok, the block was released by user-space.
1058 * Now let's open that block.
1059 * opening a block also thaws the queue.
1060 * Thawing is a side effect.
1061 */
1062 prb_open_block(pkc, pbd);
1063 }
1064 }
1065
1066 smp_mb();
1067 curr = pkc->nxt_offset;
1068 pkc->skb = skb;
e3192690 1069 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1070
1071 /* first try the current block */
1072 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1073 prb_fill_curr_block(curr, pkc, pbd, len);
1074 return (void *)curr;
1075 }
1076
1077 /* Ok, close the current block */
1078 prb_retire_current_block(pkc, po, 0);
1079
1080 /* Now, try to dispatch the next block */
1081 curr = (char *)prb_dispatch_next_block(pkc, po);
1082 if (curr) {
1083 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1084 prb_fill_curr_block(curr, pkc, pbd, len);
1085 return (void *)curr;
1086 }
1087
1088 /*
1089 * No free blocks are available.user_space hasn't caught up yet.
1090 * Queue was just frozen and now this packet will get dropped.
1091 */
1092 return NULL;
1093}
1094
eea49cc9 1095static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1096 struct sk_buff *skb,
1097 int status, unsigned int len)
1098{
1099 char *curr = NULL;
1100 switch (po->tp_version) {
1101 case TPACKET_V1:
1102 case TPACKET_V2:
1103 curr = packet_lookup_frame(po, &po->rx_ring,
1104 po->rx_ring.head, status);
1105 return curr;
1106 case TPACKET_V3:
1107 return __packet_lookup_frame_in_block(po, skb, status, len);
1108 default:
1109 WARN(1, "TPACKET version not supported\n");
1110 BUG();
99aa3473 1111 return NULL;
f6fb8f10 1112 }
1113}
1114
eea49cc9 1115static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1116 struct packet_ring_buffer *rb,
77f65ebd 1117 unsigned int idx,
f6fb8f10 1118 int status)
1119{
bc59ba39 1120 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1121 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1122
1123 if (status != BLOCK_STATUS(pbd))
1124 return NULL;
1125 return pbd;
1126}
1127
eea49cc9 1128static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1129{
1130 unsigned int prev;
1131 if (rb->prb_bdqc.kactive_blk_num)
1132 prev = rb->prb_bdqc.kactive_blk_num-1;
1133 else
1134 prev = rb->prb_bdqc.knum_blocks-1;
1135 return prev;
1136}
1137
1138/* Assumes caller has held the rx_queue.lock */
eea49cc9 1139static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1140 struct packet_ring_buffer *rb,
1141 int status)
1142{
1143 unsigned int previous = prb_previous_blk_num(rb);
1144 return prb_lookup_block(po, rb, previous, status);
1145}
1146
eea49cc9 1147static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1148 struct packet_ring_buffer *rb,
1149 int status)
1150{
1151 if (po->tp_version <= TPACKET_V2)
1152 return packet_previous_frame(po, rb, status);
1153
1154 return __prb_previous_block(po, rb, status);
1155}
1156
eea49cc9 1157static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1158 struct packet_ring_buffer *rb)
1159{
1160 switch (po->tp_version) {
1161 case TPACKET_V1:
1162 case TPACKET_V2:
1163 return packet_increment_head(rb);
1164 case TPACKET_V3:
1165 default:
1166 WARN(1, "TPACKET version not supported.\n");
1167 BUG();
1168 return;
1169 }
1170}
1171
eea49cc9 1172static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1173 struct packet_ring_buffer *rb,
1174 int status)
1175{
1176 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1177 return packet_lookup_frame(po, rb, previous, status);
1178}
1179
eea49cc9 1180static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1181{
1182 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1183}
1184
b0138408
DB		 1185static void packet_inc_pending(struct packet_ring_buffer *rb)
1186{
1187 this_cpu_inc(*rb->pending_refcnt);
1188}
1189
1190static void packet_dec_pending(struct packet_ring_buffer *rb)
1191{
1192 this_cpu_dec(*rb->pending_refcnt);
1193}
1194
1195static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1196{
1197 unsigned int refcnt = 0;
1198 int cpu;
1199
1200 /* We don't use pending refcount in rx_ring. */
1201 if (rb->pending_refcnt == NULL)
1202 return 0;
1203
1204 for_each_possible_cpu(cpu)
1205 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1206
1207 return refcnt;
1208}
1209
1210static int packet_alloc_pending(struct packet_sock *po)
1211{
1212 po->rx_ring.pending_refcnt = NULL;
1213
1214 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1215 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1216 return -ENOBUFS;
1217
1218 return 0;
1219}
1220
1221static void packet_free_pending(struct packet_sock *po)
1222{
1223 free_percpu(po->tx_ring.pending_refcnt);
1224}
1225
77f65ebd
WB		 1226static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1227{
1228 struct sock *sk = &po->sk;
1229 bool has_room;
1230
1231 if (po->prot_hook.func != tpacket_rcv)
1232 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1233 <= sk->sk_rcvbuf;
1234
1235 spin_lock(&sk->sk_receive_queue.lock);
1236 if (po->tp_version == TPACKET_V3)
1237 has_room = prb_lookup_block(po, &po->rx_ring,
1238 po->rx_ring.prb_bdqc.kactive_blk_num,
1239 TP_STATUS_KERNEL);
1240 else
1241 has_room = packet_lookup_frame(po, &po->rx_ring,
1242 po->rx_ring.head,
1243 TP_STATUS_KERNEL);
1244 spin_unlock(&sk->sk_receive_queue.lock);
1245
1246 return has_room;
1247}
1248
1da177e4
LT		 1249static void packet_sock_destruct(struct sock *sk)
1250{
ed85b565
RC		 1251 skb_queue_purge(&sk->sk_error_queue);
1252
547b792c
IJ		 1253 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1254 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1255
1256 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1257 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1258 return;
1259 }
1260
17ab56a2 1261 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1262}
1263
dc99f600
DM		 1264static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1265{
1266 int x = atomic_read(&f->rr_cur) + 1;
1267
1268 if (x >= num)
1269 x = 0;
1270
1271 return x;
1272}
1273
77f65ebd
WB		 1274static unsigned int fanout_demux_hash(struct packet_fanout *f,
1275 struct sk_buff *skb,
1276 unsigned int num)
dc99f600 1277{
61b905da 1278 return reciprocal_scale(skb_get_hash(skb), num);
dc99f600
DM		 1279}
1280
77f65ebd
WB		 1281static unsigned int fanout_demux_lb(struct packet_fanout *f,
1282 struct sk_buff *skb,
1283 unsigned int num)
dc99f600
DM		 1284{
1285 int cur, old;
1286
1287 cur = atomic_read(&f->rr_cur);
1288 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1289 fanout_rr_next(f, num))) != cur)
1290 cur = old;
77f65ebd
WB		 1291 return cur;
1292}
1293
1294static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1295 struct sk_buff *skb,
1296 unsigned int num)
1297{
1298 return smp_processor_id() % num;
dc99f600
DM		 1299}
1300
5df0ddfb
DB		 1301static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1302 struct sk_buff *skb,
1303 unsigned int num)
1304{
f337db64 1305 return prandom_u32_max(num);
5df0ddfb
DB		 1306}
1307
77f65ebd
WB		 1308static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1309 struct sk_buff *skb,
1310 unsigned int idx, unsigned int skip,
1311 unsigned int num)
95ec3eb4 1312{
77f65ebd 1313 unsigned int i, j;
95ec3eb4 1314
77f65ebd
WB		 1315 i = j = min_t(int, f->next[idx], num - 1);
1316 do {
1317 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1318 if (i != j)
1319 f->next[idx] = i;
1320 return i;
1321 }
1322 if (++i == num)
1323 i = 0;
1324 } while (i != j);
1325
1326 return idx;
1327}
1328
2d36097d
NH		 1329static unsigned int fanout_demux_qm(struct packet_fanout *f,
1330 struct sk_buff *skb,
1331 unsigned int num)
1332{
1333 return skb_get_queue_mapping(skb) % num;
1334}
1335
77f65ebd
WB		 1336static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1337{
1338 return f->flags & (flag >> 8);
95ec3eb4
DM		 1339}
1340
95ec3eb4
DM		 1341static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1342 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1343{
1344 struct packet_fanout *f = pt->af_packet_priv;
1345 unsigned int num = f->num_members;
1346 struct packet_sock *po;
77f65ebd 1347 unsigned int idx;
dc99f600
DM		 1348
1349 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1350 !num) {
1351 kfree_skb(skb);
1352 return 0;
1353 }
1354
95ec3eb4
DM		 1355 switch (f->type) {
1356 case PACKET_FANOUT_HASH:
1357 default:
77f65ebd 1358 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1359 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1360 if (!skb)
1361 return 0;
1362 }
77f65ebd 1363 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1364 break;
1365 case PACKET_FANOUT_LB:
77f65ebd 1366 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1367 break;
1368 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1369 idx = fanout_demux_cpu(f, skb, num);
1370 break;
5df0ddfb
DB		 1371 case PACKET_FANOUT_RND:
1372 idx = fanout_demux_rnd(f, skb, num);
1373 break;
2d36097d
NH		 1374 case PACKET_FANOUT_QM:
1375 idx = fanout_demux_qm(f, skb, num);
1376 break;
77f65ebd
WB		 1377 case PACKET_FANOUT_ROLLOVER:
1378 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1379 break;
dc99f600
DM		 1380 }
1381
77f65ebd
WB		 1382 po = pkt_sk(f->arr[idx]);
1383 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1384 unlikely(!packet_rcv_has_room(po, skb))) {
1385 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1386 po = pkt_sk(f->arr[idx]);
1387 }
dc99f600
DM		 1388
1389 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1390}
1391
fff3321d
PE		 1392DEFINE_MUTEX(fanout_mutex);
1393EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1394static LIST_HEAD(fanout_list);
1395
1396static void __fanout_link(struct sock *sk, struct packet_sock *po)
1397{
1398 struct packet_fanout *f = po->fanout;
1399
1400 spin_lock(&f->lock);
1401 f->arr[f->num_members] = sk;
1402 smp_wmb();
1403 f->num_members++;
1404 spin_unlock(&f->lock);
1405}
1406
1407static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1408{
1409 struct packet_fanout *f = po->fanout;
1410 int i;
1411
1412 spin_lock(&f->lock);
1413 for (i = 0; i < f->num_members; i++) {
1414 if (f->arr[i] == sk)
1415 break;
1416 }
1417 BUG_ON(i >= f->num_members);
1418 f->arr[i] = f->arr[f->num_members - 1];
1419 f->num_members--;
1420 spin_unlock(&f->lock);
1421}
1422
d4dd8aee 1423static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1424{
d4dd8aee 1425 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
c0de08d0
EL		 1426 return true;
1427
1428 return false;
1429}
1430
7736d33f 1431static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1432{
1433 struct packet_sock *po = pkt_sk(sk);
1434 struct packet_fanout *f, *match;
7736d33f 1435 u8 type = type_flags & 0xff;
77f65ebd 1436 u8 flags = type_flags >> 8;
dc99f600
DM		 1437 int err;
1438
1439 switch (type) {
77f65ebd
WB		 1440 case PACKET_FANOUT_ROLLOVER:
1441 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1442 return -EINVAL;
dc99f600
DM		 1443 case PACKET_FANOUT_HASH:
1444 case PACKET_FANOUT_LB:
95ec3eb4 1445 case PACKET_FANOUT_CPU:
5df0ddfb 1446 case PACKET_FANOUT_RND:
2d36097d 1447 case PACKET_FANOUT_QM:
dc99f600
DM		 1448 break;
1449 default:
1450 return -EINVAL;
1451 }
1452
1453 if (!po->running)
1454 return -EINVAL;
1455
1456 if (po->fanout)
1457 return -EALREADY;
1458
1459 mutex_lock(&fanout_mutex);
1460 match = NULL;
1461 list_for_each_entry(f, &fanout_list, list) {
1462 if (f->id == id &&
1463 read_pnet(&f->net) == sock_net(sk)) {
1464 match = f;
1465 break;
1466 }
1467 }
afe62c68 1468 err = -EINVAL;
77f65ebd 1469 if (match && match->flags != flags)
afe62c68 1470 goto out;
dc99f600 1471 if (!match) {
afe62c68 1472 err = -ENOMEM;
dc99f600 1473 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1474 if (!match)
1475 goto out;
1476 write_pnet(&match->net, sock_net(sk));
1477 match->id = id;
1478 match->type = type;
77f65ebd 1479 match->flags = flags;
afe62c68
ED		 1480 atomic_set(&match->rr_cur, 0);
1481 INIT_LIST_HEAD(&match->list);
1482 spin_lock_init(&match->lock);
1483 atomic_set(&match->sk_ref, 0);
1484 match->prot_hook.type = po->prot_hook.type;
1485 match->prot_hook.dev = po->prot_hook.dev;
1486 match->prot_hook.func = packet_rcv_fanout;
1487 match->prot_hook.af_packet_priv = match;
c0de08d0 1488 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1489 dev_add_pack(&match->prot_hook);
1490 list_add(&match->list, &fanout_list);
dc99f600 1491 }
afe62c68
ED		 1492 err = -EINVAL;
1493 if (match->type == type &&
1494 match->prot_hook.type == po->prot_hook.type &&
1495 match->prot_hook.dev == po->prot_hook.dev) {
1496 err = -ENOSPC;
1497 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1498 __dev_remove_pack(&po->prot_hook);
1499 po->fanout = match;
1500 atomic_inc(&match->sk_ref);
1501 __fanout_link(sk, po);
1502 err = 0;
dc99f600
DM		 1503 }
1504 }
afe62c68 1505out:
dc99f600
DM		 1506 mutex_unlock(&fanout_mutex);
1507 return err;
1508}
1509
1510static void fanout_release(struct sock *sk)
1511{
1512 struct packet_sock *po = pkt_sk(sk);
1513 struct packet_fanout *f;
1514
1515 f = po->fanout;
1516 if (!f)
1517 return;
1518
fff3321d 1519 mutex_lock(&fanout_mutex);
dc99f600
DM		 1520 po->fanout = NULL;
1521
dc99f600
DM		 1522 if (atomic_dec_and_test(&f->sk_ref)) {
1523 list_del(&f->list);
1524 dev_remove_pack(&f->prot_hook);
1525 kfree(f);
1526 }
1527 mutex_unlock(&fanout_mutex);
1528}
1da177e4 1529
90ddc4f0 1530static const struct proto_ops packet_ops;
1da177e4 1531
90ddc4f0 1532static const struct proto_ops packet_ops_spkt;
1da177e4 1533
40d4e3df
ED		 1534static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1535 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1536{
1537 struct sock *sk;
1538 struct sockaddr_pkt *spkt;
1539
1540 /*
1541 * When we registered the protocol we saved the socket in the data
1542 * field for just this event.
1543 */
1544
1545 sk = pt->af_packet_priv;
1ce4f28b 1546
1da177e4
LT		 1547 /*
1548 * Yank back the headers [hope the device set this
1549 * right or kerboom...]
1550 *
1551 * Incoming packets have ll header pulled,
1552 * push it back.
1553 *
98e399f8 1554 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1555 * so that this procedure is noop.
1556 */
1557
1558 if (skb->pkt_type == PACKET_LOOPBACK)
1559 goto out;
1560
09ad9bc7 1561 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1562 goto out;
1563
40d4e3df
ED		 1564 skb = skb_share_check(skb, GFP_ATOMIC);
1565 if (skb == NULL)
1da177e4
LT		 1566 goto oom;
1567
1568 /* drop any routing info */
adf30907 1569 skb_dst_drop(skb);
1da177e4 1570
84531c24
PO		 1571 /* drop conntrack reference */
1572 nf_reset(skb);
1573
ffbc6111 1574 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1575
98e399f8 1576 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1577
1578 /*
1579 * The SOCK_PACKET socket receives _all_ frames.
1580 */
1581
1582 spkt->spkt_family = dev->type;
1583 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1584 spkt->spkt_protocol = skb->protocol;
1585
1586 /*
1587 * Charge the memory to the socket. This is done specifically
1588 * to prevent sockets using all the memory up.
1589 */
1590
40d4e3df 1591 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1592 return 0;
1593
1594out:
1595 kfree_skb(skb);
1596oom:
1597 return 0;
1598}
1599
1600
1601/*
1602 * Output a raw packet to a device layer. This bypasses all the other
1603 * protocol layers and you must therefore supply it with a complete frame
1604 */
1ce4f28b 1605
1da177e4
LT		 1606static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1607 struct msghdr *msg, size_t len)
1608{
1609 struct sock *sk = sock->sk;
342dfc30 1610 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1611 struct sk_buff *skb = NULL;
1da177e4 1612 struct net_device *dev;
40d4e3df 1613 __be16 proto = 0;
1da177e4 1614 int err;
3bdc0eba 1615 int extra_len = 0;
1ce4f28b 1616
1da177e4 1617 /*
1ce4f28b 1618 * Get and verify the address.
1da177e4
LT		 1619 */
1620
40d4e3df 1621 if (saddr) {
1da177e4 1622 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1623 return -EINVAL;
1624 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1625 proto = saddr->spkt_protocol;
1626 } else
1627 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1628
1629 /*
1ce4f28b 1630 * Find the device first to size check it
1da177e4
LT		 1631 */
1632
de74e92a 1633 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1634retry:
654d1f8a
ED		 1635 rcu_read_lock();
1636 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1637 err = -ENODEV;
1638 if (dev == NULL)
1639 goto out_unlock;
1ce4f28b 1640
d5e76b0a
DM		 1641 err = -ENETDOWN;
1642 if (!(dev->flags & IFF_UP))
1643 goto out_unlock;
1644
1da177e4 1645 /*
40d4e3df
ED		 1646 * You may not queue a frame bigger than the mtu. This is the lowest level
1647 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1648 */
1ce4f28b 1649
3bdc0eba
BG		 1650 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1651 if (!netif_supports_nofcs(dev)) {
1652 err = -EPROTONOSUPPORT;
1653 goto out_unlock;
1654 }
1655 extra_len = 4; /* We're doing our own CRC */
1656 }
1657
1da177e4 1658 err = -EMSGSIZE;
3bdc0eba 1659 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1660 goto out_unlock;
1661
1a35ca80
ED		 1662 if (!skb) {
1663 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1664 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1665 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1666
1667 rcu_read_unlock();
4ce40912 1668 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1669 if (skb == NULL)
1670 return -ENOBUFS;
1671 /* FIXME: Save some space for broken drivers that write a hard
1672 * header at transmission time by themselves. PPP is the notable
1673 * one here. This should really be fixed at the driver level.
1674 */
1675 skb_reserve(skb, reserved);
1676 skb_reset_network_header(skb);
1677
1678 /* Try to align data part correctly */
1679 if (hhlen) {
1680 skb->data -= hhlen;
1681 skb->tail -= hhlen;
1682 if (len < hhlen)
1683 skb_reset_network_header(skb);
1684 }
1685 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1686 if (err)
1687 goto out_free;
1688 goto retry;
1da177e4
LT		 1689 }
1690
3bdc0eba 1691 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1692 /* Earlier code assumed this would be a VLAN pkt,
1693 * double-check this now that we have the actual
1694 * packet in hand.
1695 */
1696 struct ethhdr *ehdr;
1697 skb_reset_mac_header(skb);
1698 ehdr = eth_hdr(skb);
1699 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1700 err = -EMSGSIZE;
1701 goto out_unlock;
1702 }
1703 }
1a35ca80 1704
1da177e4
LT		 1705 skb->protocol = proto;
1706 skb->dev = dev;
1707 skb->priority = sk->sk_priority;
2d37a186 1708 skb->mark = sk->sk_mark;
bf84a010
DB		 1709
1710 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1711
3bdc0eba
BG		 1712 if (unlikely(extra_len == 4))
1713 skb->no_fcs = 1;
1714
40893fd0 1715 skb_probe_transport_header(skb, 0);
c1aad275 1716
1da177e4 1717 dev_queue_xmit(skb);
654d1f8a 1718 rcu_read_unlock();
40d4e3df 1719 return len;
1da177e4 1720
1da177e4 1721out_unlock:
654d1f8a 1722 rcu_read_unlock();
1a35ca80
ED		 1723out_free:
1724 kfree_skb(skb);
1da177e4
LT		 1725 return err;
1726}
1da177e4 1727
eea49cc9 1728static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1729 const struct sock *sk,
dbcb5855 1730 unsigned int res)
1da177e4
LT		 1731{
1732 struct sk_filter *filter;
fda9ef5d 1733
80f8f102
ED		 1734 rcu_read_lock();
1735 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1736 if (filter != NULL)
0a14842f 1737 res = SK_RUN_FILTER(filter, skb);
80f8f102 1738 rcu_read_unlock();
1da177e4 1739
dbcb5855 1740 return res;
1da177e4
LT		 1741}
1742
1743/*
62ab0812
ED		 1744 * This function makes lazy skb cloning in hope that most of packets
1745 * are discarded by BPF.
1746 *
1747 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1748 * and skb->cb are mangled. It works because (and until) packets
1749 * falling here are owned by current CPU. Output packets are cloned
1750 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1751 * sequencially, so that if we return skb to original state on exit,
1752 * we will not harm anyone.
1da177e4
LT		 1753 */
1754
40d4e3df
ED		 1755static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1756 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1757{
1758 struct sock *sk;
1759 struct sockaddr_ll *sll;
1760 struct packet_sock *po;
40d4e3df 1761 u8 *skb_head = skb->data;
1da177e4 1762 int skb_len = skb->len;
dbcb5855 1763 unsigned int snaplen, res;
1da177e4
LT		 1764
1765 if (skb->pkt_type == PACKET_LOOPBACK)
1766 goto drop;
1767
1768 sk = pt->af_packet_priv;
1769 po = pkt_sk(sk);
1770
09ad9bc7 1771 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1772 goto drop;
1773
1da177e4
LT		 1774 skb->dev = dev;
1775
3b04ddde 1776 if (dev->header_ops) {
1da177e4 1777 /* The device has an explicit notion of ll header,
62ab0812
ED		 1778 * exported to higher levels.
1779 *
1780 * Otherwise, the device hides details of its frame
1781 * structure, so that corresponding packet head is
1782 * never delivered to user.
1da177e4
LT		 1783 */
1784 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1785 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1786 else if (skb->pkt_type == PACKET_OUTGOING) {
1787 /* Special case: outgoing packets have ll header at head */
bbe735e4 1788 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1789 }
1790 }
1791
1792 snaplen = skb->len;
1793
dbcb5855
DM		 1794 res = run_filter(skb, sk, snaplen);
1795 if (!res)
fda9ef5d 1796 goto drop_n_restore;
dbcb5855
DM		 1797 if (snaplen > res)
1798 snaplen = res;
1da177e4 1799
0fd7bac6 1800 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1801 goto drop_n_acct;
1802
1803 if (skb_shared(skb)) {
1804 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1805 if (nskb == NULL)
1806 goto drop_n_acct;
1807
1808 if (skb_head != skb->data) {
1809 skb->data = skb_head;
1810 skb->len = skb_len;
1811 }
abc4e4fa 1812 consume_skb(skb);
1da177e4
LT		 1813 skb = nskb;
1814 }
1815
ffbc6111
HX		 1816 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1817 sizeof(skb->cb));
1818
1819 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1820 sll->sll_family = AF_PACKET;
1821 sll->sll_hatype = dev->type;
1822 sll->sll_protocol = skb->protocol;
1823 sll->sll_pkttype = skb->pkt_type;
8032b464 1824 if (unlikely(po->origdev))
80feaacb
PWJ		 1825 sll->sll_ifindex = orig_dev->ifindex;
1826 else
1827 sll->sll_ifindex = dev->ifindex;
1da177e4 1828
b95cce35 1829 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1830
ffbc6111 1831 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1832
1da177e4
LT		 1833 if (pskb_trim(skb, snaplen))
1834 goto drop_n_acct;
1835
1836 skb_set_owner_r(skb, sk);
1837 skb->dev = NULL;
adf30907 1838 skb_dst_drop(skb);
1da177e4 1839
84531c24
PO		 1840 /* drop conntrack reference */
1841 nf_reset(skb);
1842
1da177e4 1843 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1844 po->stats.stats1.tp_packets++;
3b885787 1845 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1846 __skb_queue_tail(&sk->sk_receive_queue, skb);
1847 spin_unlock(&sk->sk_receive_queue.lock);
676d2369 1848 sk->sk_data_ready(sk);
1da177e4
LT		 1849 return 0;
1850
1851drop_n_acct:
7091fbd8 1852 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1853 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1854 atomic_inc(&sk->sk_drops);
1855 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1856
1857drop_n_restore:
1858 if (skb_head != skb->data && skb_shared(skb)) {
1859 skb->data = skb_head;
1860 skb->len = skb_len;
1861 }
1862drop:
ead2ceb0 1863 consume_skb(skb);
1da177e4
LT		 1864 return 0;
1865}
1866
40d4e3df
ED		 1867static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1868 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1869{
1870 struct sock *sk;
1871 struct packet_sock *po;
1872 struct sockaddr_ll *sll;
184f489e 1873 union tpacket_uhdr h;
40d4e3df 1874 u8 *skb_head = skb->data;
1da177e4 1875 int skb_len = skb->len;
dbcb5855 1876 unsigned int snaplen, res;
f6fb8f10 1877 unsigned long status = TP_STATUS_USER;
bbd6ef87 1878 unsigned short macoff, netoff, hdrlen;
1da177e4 1879 struct sk_buff *copy_skb = NULL;
bbd6ef87 1880 struct timespec ts;
b9c32fb2 1881 __u32 ts_status;
1da177e4 1882
51846355
AW		 1883 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1884 * We may add members to them until current aligned size without forcing
1885 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1886 */
1887 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1888 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1889
1da177e4
LT		 1890 if (skb->pkt_type == PACKET_LOOPBACK)
1891 goto drop;
1892
1893 sk = pt->af_packet_priv;
1894 po = pkt_sk(sk);
1895
09ad9bc7 1896 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1897 goto drop;
1898
3b04ddde 1899 if (dev->header_ops) {
1da177e4 1900 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1901 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1902 else if (skb->pkt_type == PACKET_OUTGOING) {
1903 /* Special case: outgoing packets have ll header at head */
bbe735e4 1904 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1905 }
1906 }
1907
8dc41944
HX		 1908 if (skb->ip_summed == CHECKSUM_PARTIAL)
1909 status |= TP_STATUS_CSUMNOTREADY;
1910
1da177e4
LT		 1911 snaplen = skb->len;
1912
dbcb5855
DM		 1913 res = run_filter(skb, sk, snaplen);
1914 if (!res)
fda9ef5d 1915 goto drop_n_restore;
dbcb5855
DM		 1916 if (snaplen > res)
1917 snaplen = res;
1da177e4
LT		 1918
1919 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1920 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1921 po->tp_reserve;
1da177e4 1922 } else {
95c96174 1923 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1924 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1925 (maclen < 16 ? 16 : maclen)) +
1926 po->tp_reserve;
1da177e4
LT		 1927 macoff = netoff - maclen;
1928 }
f6fb8f10 1929 if (po->tp_version <= TPACKET_V2) {
1930 if (macoff + snaplen > po->rx_ring.frame_size) {
1931 if (po->copy_thresh &&
0fd7bac6 1932 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1933 if (skb_shared(skb)) {
1934 copy_skb = skb_clone(skb, GFP_ATOMIC);
1935 } else {
1936 copy_skb = skb_get(skb);
1937 skb_head = skb->data;
1938 }
1939 if (copy_skb)
1940 skb_set_owner_r(copy_skb, sk);
1da177e4 1941 }
f6fb8f10 1942 snaplen = po->rx_ring.frame_size - macoff;
1943 if ((int)snaplen < 0)
1944 snaplen = 0;
1da177e4 1945 }
dc808110
ED		 1946 } else if (unlikely(macoff + snaplen >
1947 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
1948 u32 nval;
1949
1950 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
1951 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
1952 snaplen, nval, macoff);
1953 snaplen = nval;
1954 if (unlikely((int)snaplen < 0)) {
1955 snaplen = 0;
1956 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
1957 }
1da177e4 1958 }
1da177e4 1959 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1960 h.raw = packet_current_rx_frame(po, skb,
1961 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1962 if (!h.raw)
1da177e4 1963 goto ring_is_full;
f6fb8f10 1964 if (po->tp_version <= TPACKET_V2) {
1965 packet_increment_rx_head(po, &po->rx_ring);
1966 /*
1967 * LOSING will be reported till you read the stats,
1968 * because it's COR - Clear On Read.
1969 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1970 * at packet level.
1971 */
ee80fbf3 1972 if (po->stats.stats1.tp_drops)
f6fb8f10 1973 status |= TP_STATUS_LOSING;
1974 }
ee80fbf3 1975 po->stats.stats1.tp_packets++;
1da177e4
LT		 1976 if (copy_skb) {
1977 status |= TP_STATUS_COPY;
1978 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1979 }
1da177e4
LT		 1980 spin_unlock(&sk->sk_receive_queue.lock);
1981
bbd6ef87 1982 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1983
1984 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1985 getnstimeofday(&ts);
1da177e4 1986
b9c32fb2
DB		 1987 status |= ts_status;
1988
bbd6ef87
PM		 1989 switch (po->tp_version) {
1990 case TPACKET_V1:
1991 h.h1->tp_len = skb->len;
1992 h.h1->tp_snaplen = snaplen;
1993 h.h1->tp_mac = macoff;
1994 h.h1->tp_net = netoff;
4b457bdf
DB		 1995 h.h1->tp_sec = ts.tv_sec;
1996 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1997 hdrlen = sizeof(*h.h1);
1998 break;
1999 case TPACKET_V2:
2000 h.h2->tp_len = skb->len;
2001 h.h2->tp_snaplen = snaplen;
2002 h.h2->tp_mac = macoff;
2003 h.h2->tp_net = netoff;
bbd6ef87
PM		 2004 h.h2->tp_sec = ts.tv_sec;
2005 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 2006 if (vlan_tx_tag_present(skb)) {
2007 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 2008 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2009 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2010 } else {
2011 h.h2->tp_vlan_tci = 0;
a0cdfcf3 2012 h.h2->tp_vlan_tpid = 0;
a3bcc23e 2013 }
e4d26f4b 2014 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 2015 hdrlen = sizeof(*h.h2);
2016 break;
f6fb8f10 2017 case TPACKET_V3:
2018 /* tp_nxt_offset,vlan are already populated above.
2019 * So DONT clear those fields here
2020 */
2021 h.h3->tp_status |= status;
2022 h.h3->tp_len = skb->len;
2023 h.h3->tp_snaplen = snaplen;
2024 h.h3->tp_mac = macoff;
2025 h.h3->tp_net = netoff;
f6fb8f10 2026 h.h3->tp_sec = ts.tv_sec;
2027 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2028 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2029 hdrlen = sizeof(*h.h3);
2030 break;
bbd6ef87
PM		 2031 default:
2032 BUG();
2033 }
1da177e4 2034
bbd6ef87 2035 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2036 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2037 sll->sll_family = AF_PACKET;
2038 sll->sll_hatype = dev->type;
2039 sll->sll_protocol = skb->protocol;
2040 sll->sll_pkttype = skb->pkt_type;
8032b464 2041 if (unlikely(po->origdev))
80feaacb
PWJ		 2042 sll->sll_ifindex = orig_dev->ifindex;
2043 else
2044 sll->sll_ifindex = dev->ifindex;
1da177e4 2045
e16aa207 2046 smp_mb();
f0d4eb29 2047
f6dafa95 2048#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2049 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2050 u8 *start, *end;
2051
f0d4eb29
DB		 2052 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2053 macoff + snaplen);
2054
2055 for (start = h.raw; start < end; start += PAGE_SIZE)
2056 flush_dcache_page(pgv_to_page(start));
1da177e4 2057 }
f0d4eb29 2058 smp_wmb();
f6dafa95 2059#endif
f0d4eb29 2060
f6fb8f10 2061 if (po->tp_version <= TPACKET_V2)
2062 __packet_set_status(po, h.raw, status);
2063 else
2064 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4 2065
676d2369 2066 sk->sk_data_ready(sk);
1da177e4
LT		 2067
2068drop_n_restore:
2069 if (skb_head != skb->data && skb_shared(skb)) {
2070 skb->data = skb_head;
2071 skb->len = skb_len;
2072 }
2073drop:
1ce4f28b 2074 kfree_skb(skb);
1da177e4
LT		 2075 return 0;
2076
2077ring_is_full:
ee80fbf3 2078 po->stats.stats1.tp_drops++;
1da177e4
LT		 2079 spin_unlock(&sk->sk_receive_queue.lock);
2080
676d2369 2081 sk->sk_data_ready(sk);
acb5d75b 2082 kfree_skb(copy_skb);
1da177e4
LT		 2083 goto drop_n_restore;
2084}
2085
69e3c75f
JB		 2086static void tpacket_destruct_skb(struct sk_buff *skb)
2087{
2088 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2089
69e3c75f 2090 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2091 void *ph;
b9c32fb2
DB		 2092 __u32 ts;
2093
69e3c75f 2094 ph = skb_shinfo(skb)->destructor_arg;
b0138408 2095 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2096
2097 ts = __packet_set_timestamp(po, ph, skb);
2098 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2099 }
2100
2101 sock_wfree(skb);
2102}
2103
40d4e3df
ED		 2104static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2105 void *frame, struct net_device *dev, int size_max,
ae641949 2106 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 2107{
184f489e 2108 union tpacket_uhdr ph;
09effa67 2109 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 2110 struct socket *sock = po->sk.sk_socket;
2111 struct page *page;
2112 void *data;
2113 int err;
2114
2115 ph.raw = frame;
2116
2117 skb->protocol = proto;
2118 skb->dev = dev;
2119 skb->priority = po->sk.sk_priority;
2d37a186 2120 skb->mark = po->sk.sk_mark;
2e31396f 2121 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2122 skb_shinfo(skb)->destructor_arg = ph.raw;
2123
2124 switch (po->tp_version) {
2125 case TPACKET_V2:
2126 tp_len = ph.h2->tp_len;
2127 break;
2128 default:
2129 tp_len = ph.h1->tp_len;
2130 break;
2131 }
09effa67
DM		 2132 if (unlikely(tp_len > size_max)) {
2133 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2134 return -EMSGSIZE;
2135 }
69e3c75f 2136
ae641949 2137 skb_reserve(skb, hlen);
69e3c75f 2138 skb_reset_network_header(skb);
c1aad275 2139
d346a3fa
DB		 2140 if (!packet_use_direct_xmit(po))
2141 skb_probe_transport_header(skb, 0);
2142 if (unlikely(po->tp_tx_has_off)) {
5920cd3a
PC		 2143 int off_min, off_max, off;
2144 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2145 off_max = po->tx_ring.frame_size - tp_len;
2146 if (sock->type == SOCK_DGRAM) {
2147 switch (po->tp_version) {
2148 case TPACKET_V2:
2149 off = ph.h2->tp_net;
2150 break;
2151 default:
2152 off = ph.h1->tp_net;
2153 break;
2154 }
2155 } else {
2156 switch (po->tp_version) {
2157 case TPACKET_V2:
2158 off = ph.h2->tp_mac;
2159 break;
2160 default:
2161 off = ph.h1->tp_mac;
2162 break;
2163 }
2164 }
2165 if (unlikely((off < off_min) || (off_max < off)))
2166 return -EINVAL;
2167 data = ph.raw + off;
2168 } else {
2169 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2170 }
69e3c75f
JB		 2171 to_write = tp_len;
2172
2173 if (sock->type == SOCK_DGRAM) {
2174 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2175 NULL, tp_len);
2176 if (unlikely(err < 0))
2177 return -EINVAL;
40d4e3df 2178 } else if (dev->hard_header_len) {
69e3c75f
JB		 2179 /* net device doesn't like empty head */
2180 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2181 pr_err("packet size is too short (%d < %d)\n",
2182 tp_len, dev->hard_header_len);
69e3c75f
JB		 2183 return -EINVAL;
2184 }
2185
2186 skb_push(skb, dev->hard_header_len);
2187 err = skb_store_bits(skb, 0, data,
2188 dev->hard_header_len);
2189 if (unlikely(err))
2190 return err;
2191
2192 data += dev->hard_header_len;
2193 to_write -= dev->hard_header_len;
2194 }
2195
69e3c75f
JB		 2196 offset = offset_in_page(data);
2197 len_max = PAGE_SIZE - offset;
2198 len = ((to_write > len_max) ? len_max : to_write);
2199
2200 skb->data_len = to_write;
2201 skb->len += to_write;
2202 skb->truesize += to_write;
2203 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2204
2205 while (likely(to_write)) {
2206 nr_frags = skb_shinfo(skb)->nr_frags;
2207
2208 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2209 pr_err("Packet exceed the number of skb frags(%lu)\n",
2210 MAX_SKB_FRAGS);
69e3c75f
JB		 2211 return -EFAULT;
2212 }
2213
0af55bb5
CG		 2214 page = pgv_to_page(data);
2215 data += len;
69e3c75f
JB		 2216 flush_dcache_page(page);
2217 get_page(page);
0af55bb5 2218 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2219 to_write -= len;
2220 offset = 0;
2221 len_max = PAGE_SIZE;
2222 len = ((to_write > len_max) ? len_max : to_write);
2223 }
2224
2225 return tp_len;
2226}
2227
2228static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2229{
69e3c75f
JB		 2230 struct sk_buff *skb;
2231 struct net_device *dev;
2232 __be16 proto;
09effa67 2233 int err, reserve = 0;
40d4e3df 2234 void *ph;
342dfc30 2235 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2236 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
69e3c75f
JB		 2237 int tp_len, size_max;
2238 unsigned char *addr;
2239 int len_sum = 0;
9e67030a 2240 int status = TP_STATUS_AVAILABLE;
ae641949 2241 int hlen, tlen;
69e3c75f 2242
69e3c75f
JB		 2243 mutex_lock(&po->pg_vec_lock);
2244
66e56cd4 2245 if (likely(saddr == NULL)) {
e40526cb 2246 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2247 proto = po->num;
2248 addr = NULL;
2249 } else {
2250 err = -EINVAL;
2251 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2252 goto out;
2253 if (msg->msg_namelen < (saddr->sll_halen
2254 + offsetof(struct sockaddr_ll,
2255 sll_addr)))
2256 goto out;
69e3c75f
JB		 2257 proto = saddr->sll_protocol;
2258 addr = saddr->sll_addr;
827d9780 2259 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2260 }
2261
69e3c75f
JB		 2262 err = -ENXIO;
2263 if (unlikely(dev == NULL))
2264 goto out;
69e3c75f
JB		 2265 err = -ENETDOWN;
2266 if (unlikely(!(dev->flags & IFF_UP)))
2267 goto out_put;
2268
52f1454f 2269 reserve = dev->hard_header_len + VLAN_HLEN;
69e3c75f 2270 size_max = po->tx_ring.frame_size
b5dd884e 2271 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2272
09effa67
DM		 2273 if (size_max > dev->mtu + reserve)
2274 size_max = dev->mtu + reserve;
2275
69e3c75f
JB		 2276 do {
2277 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2278 TP_STATUS_SEND_REQUEST);
69e3c75f 2279 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2280 if (need_wait && need_resched())
2281 schedule();
69e3c75f
JB		 2282 continue;
2283 }
2284
2285 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2286 hlen = LL_RESERVED_SPACE(dev);
2287 tlen = dev->needed_tailroom;
69e3c75f 2288 skb = sock_alloc_send_skb(&po->sk,
ae641949 2289 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2290 0, &err);
2291
2292 if (unlikely(skb == NULL))
2293 goto out_status;
2294
2295 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
52f1454f
DB		 2296 addr, hlen);
2297 if (tp_len > dev->mtu + dev->hard_header_len) {
2298 struct ethhdr *ehdr;
2299 /* Earlier code assumed this would be a VLAN pkt,
2300 * double-check this now that we have the actual
2301 * packet in hand.
2302 */
69e3c75f 2303
52f1454f
DB		 2304 skb_reset_mac_header(skb);
2305 ehdr = eth_hdr(skb);
2306 if (ehdr->h_proto != htons(ETH_P_8021Q))
2307 tp_len = -EMSGSIZE;
2308 }
69e3c75f
JB		 2309 if (unlikely(tp_len < 0)) {
2310 if (po->tp_loss) {
2311 __packet_set_status(po, ph,
2312 TP_STATUS_AVAILABLE);
2313 packet_increment_head(&po->tx_ring);
2314 kfree_skb(skb);
2315 continue;
2316 } else {
2317 status = TP_STATUS_WRONG_FORMAT;
2318 err = tp_len;
2319 goto out_status;
2320 }
2321 }
2322
0fd5d57b
DB		 2323 packet_pick_tx_queue(dev, skb);
2324
69e3c75f
JB		 2325 skb->destructor = tpacket_destruct_skb;
2326 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2327 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2328
2329 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2330 err = po->xmit(skb);
eb70df13
JP		 2331 if (unlikely(err > 0)) {
2332 err = net_xmit_errno(err);
2333 if (err && __packet_get_status(po, ph) ==
2334 TP_STATUS_AVAILABLE) {
2335 /* skb was destructed already */
2336 skb = NULL;
2337 goto out_status;
2338 }
2339 /*
2340 * skb was dropped but not destructed yet;
2341 * let's treat it like congestion or err < 0
2342 */
2343 err = 0;
2344 }
69e3c75f
JB		 2345 packet_increment_head(&po->tx_ring);
2346 len_sum += tp_len;
b0138408
DB		 2347 } while (likely((ph != NULL) ||
2348 /* Note: packet_read_pending() might be slow if we have
2349 * to call it as it's per_cpu variable, but in fast-path
2350 * we already short-circuit the loop with the first
2351 * condition, and luckily don't have to go that path
2352 * anyway.
2353 */
2354 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2355
2356 err = len_sum;
2357 goto out_put;
2358
69e3c75f
JB		 2359out_status:
2360 __packet_set_status(po, ph, status);
2361 kfree_skb(skb);
2362out_put:
e40526cb 2363 dev_put(dev);
69e3c75f
JB		 2364out:
2365 mutex_unlock(&po->pg_vec_lock);
2366 return err;
2367}
69e3c75f 2368
eea49cc9
OJ		 2369static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2370 size_t reserve, size_t len,
2371 size_t linear, int noblock,
2372 int *err)
bfd5f4a3
SS		 2373{
2374 struct sk_buff *skb;
2375
2376 /* Under a page? Don't bother with paged skb. */
2377 if (prepad + len < PAGE_SIZE || !linear)
2378 linear = len;
2379
2380 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2381 err, 0);
bfd5f4a3
SS		 2382 if (!skb)
2383 return NULL;
2384
2385 skb_reserve(skb, reserve);
2386 skb_put(skb, linear);
2387 skb->data_len = len - linear;
2388 skb->len += len - linear;
2389
2390 return skb;
2391}
2392
d346a3fa 2393static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2394{
2395 struct sock *sk = sock->sk;
342dfc30 2396 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2397 struct sk_buff *skb;
2398 struct net_device *dev;
0e11c91e 2399 __be16 proto;
1da177e4 2400 unsigned char *addr;
827d9780 2401 int err, reserve = 0;
bfd5f4a3
SS		 2402 struct virtio_net_hdr vnet_hdr = { 0 };
2403 int offset = 0;
2404 int vnet_hdr_len;
2405 struct packet_sock *po = pkt_sk(sk);
2406 unsigned short gso_type = 0;
ae641949 2407 int hlen, tlen;
3bdc0eba 2408 int extra_len = 0;
1da177e4
LT		 2409
2410 /*
1ce4f28b 2411 * Get and verify the address.
1da177e4 2412 */
1ce4f28b 2413
66e56cd4 2414 if (likely(saddr == NULL)) {
e40526cb 2415 dev = packet_cached_dev_get(po);
1da177e4
LT		 2416 proto = po->num;
2417 addr = NULL;
2418 } else {
2419 err = -EINVAL;
2420 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2421 goto out;
0fb375fb
EB		 2422 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2423 goto out;
1da177e4
LT		 2424 proto = saddr->sll_protocol;
2425 addr = saddr->sll_addr;
827d9780 2426 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2427 }
2428
1da177e4 2429 err = -ENXIO;
e40526cb 2430 if (unlikely(dev == NULL))
1da177e4 2431 goto out_unlock;
d5e76b0a 2432 err = -ENETDOWN;
e40526cb 2433 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2434 goto out_unlock;
2435
e40526cb
DB		 2436 if (sock->type == SOCK_RAW)
2437 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2438 if (po->has_vnet_hdr) {
2439 vnet_hdr_len = sizeof(vnet_hdr);
2440
2441 err = -EINVAL;
2442 if (len < vnet_hdr_len)
2443 goto out_unlock;
2444
2445 len -= vnet_hdr_len;
2446
2447 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2448 vnet_hdr_len);
2449 if (err < 0)
2450 goto out_unlock;
2451
2452 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2453 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2454 vnet_hdr.hdr_len))
2455 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2456 vnet_hdr.csum_offset + 2;
2457
2458 err = -EINVAL;
2459 if (vnet_hdr.hdr_len > len)
2460 goto out_unlock;
2461
2462 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2463 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2464 case VIRTIO_NET_HDR_GSO_TCPV4:
2465 gso_type = SKB_GSO_TCPV4;
2466 break;
2467 case VIRTIO_NET_HDR_GSO_TCPV6:
2468 gso_type = SKB_GSO_TCPV6;
2469 break;
2470 case VIRTIO_NET_HDR_GSO_UDP:
2471 gso_type = SKB_GSO_UDP;
2472 break;
2473 default:
2474 goto out_unlock;
2475 }
2476
2477 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2478 gso_type |= SKB_GSO_TCP_ECN;
2479
2480 if (vnet_hdr.gso_size == 0)
2481 goto out_unlock;
2482
2483 }
2484 }
2485
3bdc0eba
BG		 2486 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2487 if (!netif_supports_nofcs(dev)) {
2488 err = -EPROTONOSUPPORT;
2489 goto out_unlock;
2490 }
2491 extra_len = 4; /* We're doing our own CRC */
2492 }
2493
1da177e4 2494 err = -EMSGSIZE;
3bdc0eba 2495 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2496 goto out_unlock;
2497
bfd5f4a3 2498 err = -ENOBUFS;
ae641949
HX		 2499 hlen = LL_RESERVED_SPACE(dev);
2500 tlen = dev->needed_tailroom;
2501 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2502 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2503 if (skb == NULL)
1da177e4
LT		 2504 goto out_unlock;
2505
bfd5f4a3 2506 skb_set_network_header(skb, reserve);
1da177e4 2507
0c4e8581
SH		 2508 err = -EINVAL;
2509 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2510 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2511 goto out_free;
1da177e4
LT		 2512
2513 /* Returns -EFAULT on error */
bfd5f4a3 2514 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2515 if (err)
2516 goto out_free;
bf84a010
DB		 2517
2518 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2519
3bdc0eba 2520 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2521 /* Earlier code assumed this would be a VLAN pkt,
2522 * double-check this now that we have the actual
2523 * packet in hand.
2524 */
2525 struct ethhdr *ehdr;
2526 skb_reset_mac_header(skb);
2527 ehdr = eth_hdr(skb);
2528 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2529 err = -EMSGSIZE;
2530 goto out_free;
2531 }
57f89bfa
BG		 2532 }
2533
09effa67
DM		 2534 skb->protocol = proto;
2535 skb->dev = dev;
1da177e4 2536 skb->priority = sk->sk_priority;
2d37a186 2537 skb->mark = sk->sk_mark;
0fd5d57b
DB		 2538
2539 packet_pick_tx_queue(dev, skb);
1da177e4 2540
bfd5f4a3
SS		 2541 if (po->has_vnet_hdr) {
2542 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2543 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2544 vnet_hdr.csum_offset)) {
2545 err = -EINVAL;
2546 goto out_free;
2547 }
2548 }
2549
2550 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2551 skb_shinfo(skb)->gso_type = gso_type;
2552
2553 /* Header must be checked, and gso_segs computed. */
2554 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2555 skb_shinfo(skb)->gso_segs = 0;
2556
2557 len += vnet_hdr_len;
2558 }
2559
d346a3fa
DB		 2560 if (!packet_use_direct_xmit(po))
2561 skb_probe_transport_header(skb, reserve);
3bdc0eba
BG		 2562 if (unlikely(extra_len == 4))
2563 skb->no_fcs = 1;
2564
d346a3fa 2565 err = po->xmit(skb);
1da177e4
LT		 2566 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2567 goto out_unlock;
2568
e40526cb 2569 dev_put(dev);
1da177e4 2570
40d4e3df 2571 return len;
1da177e4
LT		 2572
2573out_free:
2574 kfree_skb(skb);
2575out_unlock:
e40526cb 2576 if (dev)
1da177e4
LT		 2577 dev_put(dev);
2578out:
2579 return err;
2580}
2581
69e3c75f
JB		 2582static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2583 struct msghdr *msg, size_t len)
2584{
69e3c75f
JB		 2585 struct sock *sk = sock->sk;
2586 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2587
69e3c75f
JB		 2588 if (po->tx_ring.pg_vec)
2589 return tpacket_snd(po, msg);
2590 else
69e3c75f
JB		 2591 return packet_snd(sock, msg, len);
2592}
2593
1da177e4
LT		 2594/*
2595 * Close a PACKET socket. This is fairly simple. We immediately go
2596 * to 'closed' state and remove our protocol entry in the device list.
2597 */
2598
2599static int packet_release(struct socket *sock)
2600{
2601 struct sock *sk = sock->sk;
2602 struct packet_sock *po;
d12d01d6 2603 struct net *net;
f6fb8f10 2604 union tpacket_req_u req_u;
1da177e4
LT		 2605
2606 if (!sk)
2607 return 0;
2608
3b1e0a65 2609 net = sock_net(sk);
1da177e4
LT		 2610 po = pkt_sk(sk);
2611
0fa7fa98 2612 mutex_lock(&net->packet.sklist_lock);
808f5114 2613 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2614 mutex_unlock(&net->packet.sklist_lock);
2615
2616 preempt_disable();
920de804 2617 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2618 preempt_enable();
1da177e4 2619
808f5114 2620 spin_lock(&po->bind_lock);
ce06b03e 2621 unregister_prot_hook(sk, false);
66e56cd4
DB		 2622 packet_cached_dev_reset(po);
2623
160ff18a
BG		 2624 if (po->prot_hook.dev) {
2625 dev_put(po->prot_hook.dev);
2626 po->prot_hook.dev = NULL;
2627 }
808f5114 2628 spin_unlock(&po->bind_lock);
1da177e4 2629
1da177e4 2630 packet_flush_mclist(sk);
1da177e4 2631
9665d5d6
PS		 2632 if (po->rx_ring.pg_vec) {
2633 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2634 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2635 }
69e3c75f 2636
9665d5d6
PS		 2637 if (po->tx_ring.pg_vec) {
2638 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2639 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2640 }
1da177e4 2641
dc99f600
DM		 2642 fanout_release(sk);
2643
808f5114 2644 synchronize_net();
1da177e4
LT		 2645 /*
2646 * Now the socket is dead. No more input will appear.
2647 */
1da177e4
LT		 2648 sock_orphan(sk);
2649 sock->sk = NULL;
2650
2651 /* Purge queues */
2652
2653 skb_queue_purge(&sk->sk_receive_queue);
b0138408 2654 packet_free_pending(po);
17ab56a2 2655 sk_refcnt_debug_release(sk);
1da177e4
LT		 2656
2657 sock_put(sk);
2658 return 0;
2659}
2660
2661/*
2662 * Attach a packet hook.
2663 */
2664
902fefb8 2665static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
1da177e4
LT		 2666{
2667 struct packet_sock *po = pkt_sk(sk);
902fefb8
DB		 2668 const struct net_device *dev_curr;
2669 __be16 proto_curr;
2670 bool need_rehook;
dc99f600 2671
aef950b4
WY		 2672 if (po->fanout) {
2673 if (dev)
2674 dev_put(dev);
2675
dc99f600 2676 return -EINVAL;
aef950b4 2677 }
1da177e4
LT		 2678
2679 lock_sock(sk);
1da177e4 2680 spin_lock(&po->bind_lock);
66e56cd4 2681
902fefb8
DB		 2682 proto_curr = po->prot_hook.type;
2683 dev_curr = po->prot_hook.dev;
2684
2685 need_rehook = proto_curr != proto || dev_curr != dev;
2686
2687 if (need_rehook) {
2688 unregister_prot_hook(sk, true);
1da177e4 2689
902fefb8
DB		 2690 po->num = proto;
2691 po->prot_hook.type = proto;
1da177e4 2692
902fefb8
DB		 2693 if (po->prot_hook.dev)
2694 dev_put(po->prot_hook.dev);
2695
2696 po->prot_hook.dev = dev;
2697
2698 po->ifindex = dev ? dev->ifindex : 0;
2699 packet_cached_dev_assign(po, dev);
2700 }
66e56cd4 2701
902fefb8 2702 if (proto == 0 || !need_rehook)
1da177e4
LT		 2703 goto out_unlock;
2704
be85d4ad 2705 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2706 register_prot_hook(sk);
be85d4ad
UT		 2707 } else {
2708 sk->sk_err = ENETDOWN;
2709 if (!sock_flag(sk, SOCK_DEAD))
2710 sk->sk_error_report(sk);
1da177e4
LT		 2711 }
2712
2713out_unlock:
2714 spin_unlock(&po->bind_lock);
2715 release_sock(sk);
2716 return 0;
2717}
2718
2719/*
2720 * Bind a packet socket to a device
2721 */
2722
40d4e3df
ED		 2723static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2724 int addr_len)
1da177e4 2725{
40d4e3df 2726 struct sock *sk = sock->sk;
1da177e4
LT		 2727 char name[15];
2728 struct net_device *dev;
2729 int err = -ENODEV;
1ce4f28b 2730
1da177e4
LT		 2731 /*
2732 * Check legality
2733 */
1ce4f28b 2734
8ae55f04 2735 if (addr_len != sizeof(struct sockaddr))
1da177e4 2736 return -EINVAL;
40d4e3df 2737 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2738
3b1e0a65 2739 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2740 if (dev)
1da177e4 2741 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2742 return err;
2743}
1da177e4
LT		 2744
2745static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2746{
40d4e3df
ED		 2747 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2748 struct sock *sk = sock->sk;
1da177e4
LT		 2749 struct net_device *dev = NULL;
2750 int err;
2751
2752
2753 /*
2754 * Check legality
2755 */
1ce4f28b 2756
1da177e4
LT		 2757 if (addr_len < sizeof(struct sockaddr_ll))
2758 return -EINVAL;
2759 if (sll->sll_family != AF_PACKET)
2760 return -EINVAL;
2761
2762 if (sll->sll_ifindex) {
2763 err = -ENODEV;
3b1e0a65 2764 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2765 if (dev == NULL)
2766 goto out;
2767 }
2768 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2769
2770out:
2771 return err;
2772}
2773
2774static struct proto packet_proto = {
2775 .name = "PACKET",
2776 .owner = THIS_MODULE,
2777 .obj_size = sizeof(struct packet_sock),
2778};
2779
2780/*
1ce4f28b 2781 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2782 */
2783
3f378b68
EP		 2784static int packet_create(struct net *net, struct socket *sock, int protocol,
2785 int kern)
1da177e4
LT		 2786{
2787 struct sock *sk;
2788 struct packet_sock *po;
0e11c91e 2789 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2790 int err;
2791
df008c91 2792 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2793 return -EPERM;
be02097c
DM		 2794 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2795 sock->type != SOCK_PACKET)
1da177e4
LT		 2796 return -ESOCKTNOSUPPORT;
2797
2798 sock->state = SS_UNCONNECTED;
2799
2800 err = -ENOBUFS;
6257ff21 2801 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2802 if (sk == NULL)
2803 goto out;
2804
2805 sock->ops = &packet_ops;
1da177e4
LT		 2806 if (sock->type == SOCK_PACKET)
2807 sock->ops = &packet_ops_spkt;
be02097c 2808
1da177e4
LT		 2809 sock_init_data(sock, sk);
2810
2811 po = pkt_sk(sk);
2812 sk->sk_family = PF_PACKET;
0e11c91e 2813 po->num = proto;
d346a3fa 2814 po->xmit = dev_queue_xmit;
66e56cd4 2815
b0138408
DB		 2816 err = packet_alloc_pending(po);
2817 if (err)
2818 goto out2;
2819
66e56cd4 2820 packet_cached_dev_reset(po);
1da177e4
LT		 2821
2822 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2823 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2824
2825 /*
2826 * Attach a protocol block
2827 */
2828
2829 spin_lock_init(&po->bind_lock);
905db440 2830 mutex_init(&po->pg_vec_lock);
1da177e4 2831 po->prot_hook.func = packet_rcv;
be02097c 2832
1da177e4
LT		 2833 if (sock->type == SOCK_PACKET)
2834 po->prot_hook.func = packet_rcv_spkt;
be02097c 2835
1da177e4
LT		 2836 po->prot_hook.af_packet_priv = sk;
2837
0e11c91e
AV		 2838 if (proto) {
2839 po->prot_hook.type = proto;
ce06b03e 2840 register_prot_hook(sk);
1da177e4
LT		 2841 }
2842
0fa7fa98 2843 mutex_lock(&net->packet.sklist_lock);
808f5114 2844 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2845 mutex_unlock(&net->packet.sklist_lock);
2846
2847 preempt_disable();
3680453c 2848 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2849 preempt_enable();
808f5114 2850
40d4e3df 2851 return 0;
b0138408
DB		 2852out2:
2853 sk_free(sk);
1da177e4
LT		 2854out:
2855 return err;
2856}
2857
2858/*
2859 * Pull a packet from our receive queue and hand it to the user.
2860 * If necessary we block.
2861 */
2862
2863static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2864 struct msghdr *msg, size_t len, int flags)
2865{
2866 struct sock *sk = sock->sk;
2867 struct sk_buff *skb;
2868 int copied, err;
bfd5f4a3 2869 int vnet_hdr_len = 0;
1da177e4
LT		 2870
2871 err = -EINVAL;
ed85b565 2872 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2873 goto out;
2874
2875#if 0
2876 /* What error should we return now? EUNATTACH? */
2877 if (pkt_sk(sk)->ifindex < 0)
2878 return -ENODEV;
2879#endif
2880
ed85b565 2881 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2882 err = sock_recv_errqueue(sk, msg, len,
2883 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2884 goto out;
2885 }
2886
1da177e4
LT		 2887 /*
2888 * Call the generic datagram receiver. This handles all sorts
2889 * of horrible races and re-entrancy so we can forget about it
2890 * in the protocol layers.
2891 *
2892 * Now it will return ENETDOWN, if device have just gone down,
2893 * but then it will block.
2894 */
2895
40d4e3df 2896 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2897
2898 /*
1ce4f28b 2899 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2900 * handles the blocking we don't see and worry about blocking
2901 * retries.
2902 */
2903
8ae55f04 2904 if (skb == NULL)
1da177e4
LT		 2905 goto out;
2906
bfd5f4a3
SS		 2907 if (pkt_sk(sk)->has_vnet_hdr) {
2908 struct virtio_net_hdr vnet_hdr = { 0 };
2909
2910 err = -EINVAL;
2911 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2912 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2913 goto out_free;
2914
1f18b717
MK		 2915 len -= vnet_hdr_len;
2916
bfd5f4a3
SS		 2917 if (skb_is_gso(skb)) {
2918 struct skb_shared_info *sinfo = skb_shinfo(skb);
2919
2920 /* This is a hint as to how much should be linear. */
2921 vnet_hdr.hdr_len = skb_headlen(skb);
2922 vnet_hdr.gso_size = sinfo->gso_size;
2923 if (sinfo->gso_type & SKB_GSO_TCPV4)
2924 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2925 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2926 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2927 else if (sinfo->gso_type & SKB_GSO_UDP)
2928 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2929 else if (sinfo->gso_type & SKB_GSO_FCOE)
2930 goto out_free;
2931 else
2932 BUG();
2933 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2934 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2935 } else
2936 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2937
2938 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2939 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2940 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2941 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2942 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2943 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2944 } /* else everything is zero */
2945
2946 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2947 vnet_hdr_len);
2948 if (err < 0)
2949 goto out_free;
2950 }
2951
f3d33426
HFS		 2952 /* You lose any data beyond the buffer you gave. If it worries
2953 * a user program they can ask the device for its MTU
2954 * anyway.
1da177e4 2955 */
1da177e4 2956 copied = skb->len;
40d4e3df
ED		 2957 if (copied > len) {
2958 copied = len;
2959 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2960 }
2961
2962 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2963 if (err)
2964 goto out_free;
2965
3b885787 2966 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2967
f3d33426
HFS		 2968 if (msg->msg_name) {
2969 /* If the address length field is there to be filled
2970 * in, we fill it in now.
2971 */
2972 if (sock->type == SOCK_PACKET) {
342dfc30 2973 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426
HFS		 2974 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2975 } else {
2976 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2977 msg->msg_namelen = sll->sll_halen +
2978 offsetof(struct sockaddr_ll, sll_addr);
2979 }
ffbc6111
HX		 2980 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2981 msg->msg_namelen);
f3d33426 2982 }
1da177e4 2983
8dc41944 2984 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2985 struct tpacket_auxdata aux;
2986
2987 aux.tp_status = TP_STATUS_USER;
2988 if (skb->ip_summed == CHECKSUM_PARTIAL)
2989 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2990 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2991 aux.tp_snaplen = skb->len;
2992 aux.tp_mac = 0;
bbe735e4 2993 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2994 if (vlan_tx_tag_present(skb)) {
2995 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 2996 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
2997 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2998 } else {
2999 aux.tp_vlan_tci = 0;
a0cdfcf3 3000 aux.tp_vlan_tpid = 0;
a3bcc23e 3001 }
ffbc6111 3002 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 3003 }
3004
1da177e4
LT		 3005 /*
3006 * Free or return the buffer as appropriate. Again this
3007 * hides all the races and re-entrancy issues from us.
3008 */
bfd5f4a3 3009 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 3010
3011out_free:
3012 skb_free_datagram(sk, skb);
3013out:
3014 return err;
3015}
3016
1da177e4
LT		 3017static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3018 int *uaddr_len, int peer)
3019{
3020 struct net_device *dev;
3021 struct sock *sk = sock->sk;
3022
3023 if (peer)
3024 return -EOPNOTSUPP;
3025
3026 uaddr->sa_family = AF_PACKET;
2dc85bf3 3027 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 3028 rcu_read_lock();
3029 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3030 if (dev)
2dc85bf3 3031 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 3032 rcu_read_unlock();
1da177e4
LT		 3033 *uaddr_len = sizeof(*uaddr);
3034
3035 return 0;
3036}
1da177e4
LT		 3037
3038static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3039 int *uaddr_len, int peer)
3040{
3041 struct net_device *dev;
3042 struct sock *sk = sock->sk;
3043 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3044 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3045
3046 if (peer)
3047 return -EOPNOTSUPP;
3048
3049 sll->sll_family = AF_PACKET;
3050 sll->sll_ifindex = po->ifindex;
3051 sll->sll_protocol = po->num;
67286640 3052 sll->sll_pkttype = 0;
654d1f8a
ED		 3053 rcu_read_lock();
3054 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3055 if (dev) {
3056 sll->sll_hatype = dev->type;
3057 sll->sll_halen = dev->addr_len;
3058 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3059 } else {
3060 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3061 sll->sll_halen = 0;
3062 }
654d1f8a 3063 rcu_read_unlock();
0fb375fb 3064 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3065
3066 return 0;
3067}
3068
2aeb0b88
WC		 3069static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3070 int what)
1da177e4
LT		 3071{
3072 switch (i->type) {
3073 case PACKET_MR_MULTICAST:
1162563f
JP		 3074 if (i->alen != dev->addr_len)
3075 return -EINVAL;
1da177e4 3076 if (what > 0)
22bedad3 3077 return dev_mc_add(dev, i->addr);
1da177e4 3078 else
22bedad3 3079 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3080 break;
3081 case PACKET_MR_PROMISC:
2aeb0b88 3082 return dev_set_promiscuity(dev, what);
1da177e4 3083 case PACKET_MR_ALLMULTI:
2aeb0b88 3084 return dev_set_allmulti(dev, what);
d95ed927 3085 case PACKET_MR_UNICAST:
1162563f
JP		 3086 if (i->alen != dev->addr_len)
3087 return -EINVAL;
d95ed927 3088 if (what > 0)
a748ee24 3089 return dev_uc_add(dev, i->addr);
d95ed927 3090 else
a748ee24 3091 return dev_uc_del(dev, i->addr);
d95ed927 3092 break;
40d4e3df
ED		 3093 default:
3094 break;
1da177e4 3095 }
2aeb0b88 3096 return 0;
1da177e4
LT		 3097}
3098
3099static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
3100{
40d4e3df 3101 for ( ; i; i = i->next) {
1da177e4
LT		 3102 if (i->ifindex == dev->ifindex)
3103 packet_dev_mc(dev, i, what);
3104 }
3105}
3106
0fb375fb 3107static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3108{
3109 struct packet_sock *po = pkt_sk(sk);
3110 struct packet_mclist *ml, *i;
3111 struct net_device *dev;
3112 int err;
3113
3114 rtnl_lock();
3115
3116 err = -ENODEV;
3b1e0a65 3117 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3118 if (!dev)
3119 goto done;
3120
3121 err = -EINVAL;
1162563f 3122 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3123 goto done;
3124
3125 err = -ENOBUFS;
8b3a7005 3126 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3127 if (i == NULL)
3128 goto done;
3129
3130 err = 0;
3131 for (ml = po->mclist; ml; ml = ml->next) {
3132 if (ml->ifindex == mreq->mr_ifindex &&
3133 ml->type == mreq->mr_type &&
3134 ml->alen == mreq->mr_alen &&
3135 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3136 ml->count++;
3137 /* Free the new element ... */
3138 kfree(i);
3139 goto done;
3140 }
3141 }
3142
3143 i->type = mreq->mr_type;
3144 i->ifindex = mreq->mr_ifindex;
3145 i->alen = mreq->mr_alen;
3146 memcpy(i->addr, mreq->mr_address, i->alen);
3147 i->count = 1;
3148 i->next = po->mclist;
3149 po->mclist = i;
2aeb0b88
WC		 3150 err = packet_dev_mc(dev, i, 1);
3151 if (err) {
3152 po->mclist = i->next;
3153 kfree(i);
3154 }
1da177e4
LT		 3155
3156done:
3157 rtnl_unlock();
3158 return err;
3159}
3160
0fb375fb 3161static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3162{
3163 struct packet_mclist *ml, **mlp;
3164
3165 rtnl_lock();
3166
3167 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3168 if (ml->ifindex == mreq->mr_ifindex &&
3169 ml->type == mreq->mr_type &&
3170 ml->alen == mreq->mr_alen &&
3171 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3172 if (--ml->count == 0) {
3173 struct net_device *dev;
3174 *mlp = ml->next;
ad959e76
ED		 3175 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3176 if (dev)
1da177e4 3177 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3178 kfree(ml);
3179 }
3180 rtnl_unlock();
3181 return 0;
3182 }
3183 }
3184 rtnl_unlock();
3185 return -EADDRNOTAVAIL;
3186}
3187
3188static void packet_flush_mclist(struct sock *sk)
3189{
3190 struct packet_sock *po = pkt_sk(sk);
3191 struct packet_mclist *ml;
3192
3193 if (!po->mclist)
3194 return;
3195
3196 rtnl_lock();
3197 while ((ml = po->mclist) != NULL) {
3198 struct net_device *dev;
3199
3200 po->mclist = ml->next;
ad959e76
ED		 3201 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3202 if (dev != NULL)
1da177e4 3203 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3204 kfree(ml);
3205 }
3206 rtnl_unlock();
3207}
1da177e4
LT		 3208
3209static int
b7058842 3210packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3211{
3212 struct sock *sk = sock->sk;
8dc41944 3213 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3214 int ret;
3215
3216 if (level != SOL_PACKET)
3217 return -ENOPROTOOPT;
3218
69e3c75f 3219 switch (optname) {
1ce4f28b 3220 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3221 case PACKET_DROP_MEMBERSHIP:
3222 {
0fb375fb
EB		 3223 struct packet_mreq_max mreq;
3224 int len = optlen;
3225 memset(&mreq, 0, sizeof(mreq));
3226 if (len < sizeof(struct packet_mreq))
1da177e4 3227 return -EINVAL;
0fb375fb
EB		 3228 if (len > sizeof(mreq))
3229 len = sizeof(mreq);
40d4e3df 3230 if (copy_from_user(&mreq, optval, len))
1da177e4 3231 return -EFAULT;
0fb375fb
EB		 3232 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3233 return -EINVAL;
1da177e4
LT		 3234 if (optname == PACKET_ADD_MEMBERSHIP)
3235 ret = packet_mc_add(sk, &mreq);
3236 else
3237 ret = packet_mc_drop(sk, &mreq);
3238 return ret;
3239 }
a2efcfa0 3240
1da177e4 3241 case PACKET_RX_RING:
69e3c75f 3242 case PACKET_TX_RING:
1da177e4 3243 {
f6fb8f10 3244 union tpacket_req_u req_u;
3245 int len;
1da177e4 3246
f6fb8f10 3247 switch (po->tp_version) {
3248 case TPACKET_V1:
3249 case TPACKET_V2:
3250 len = sizeof(req_u.req);
3251 break;
3252 case TPACKET_V3:
3253 default:
3254 len = sizeof(req_u.req3);
3255 break;
3256 }
3257 if (optlen < len)
1da177e4 3258 return -EINVAL;
bfd5f4a3
SS		 3259 if (pkt_sk(sk)->has_vnet_hdr)
3260 return -EINVAL;
f6fb8f10 3261 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3262 return -EFAULT;
f6fb8f10 3263 return packet_set_ring(sk, &req_u, 0,
3264 optname == PACKET_TX_RING);
1da177e4
LT		 3265 }
3266 case PACKET_COPY_THRESH:
3267 {
3268 int val;
3269
40d4e3df 3270 if (optlen != sizeof(val))
1da177e4 3271 return -EINVAL;
40d4e3df 3272 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3273 return -EFAULT;
3274
3275 pkt_sk(sk)->copy_thresh = val;
3276 return 0;
3277 }
bbd6ef87
PM		 3278 case PACKET_VERSION:
3279 {
3280 int val;
3281
3282 if (optlen != sizeof(val))
3283 return -EINVAL;
69e3c75f 3284 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3285 return -EBUSY;
3286 if (copy_from_user(&val, optval, sizeof(val)))
3287 return -EFAULT;
3288 switch (val) {
3289 case TPACKET_V1:
3290 case TPACKET_V2:
f6fb8f10 3291 case TPACKET_V3:
bbd6ef87
PM		 3292 po->tp_version = val;
3293 return 0;
3294 default:
3295 return -EINVAL;
3296 }
3297 }
8913336a
PM		 3298 case PACKET_RESERVE:
3299 {
3300 unsigned int val;
3301
3302 if (optlen != sizeof(val))
3303 return -EINVAL;
69e3c75f 3304 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3305 return -EBUSY;
3306 if (copy_from_user(&val, optval, sizeof(val)))
3307 return -EFAULT;
3308 po->tp_reserve = val;
3309 return 0;
3310 }
69e3c75f
JB		 3311 case PACKET_LOSS:
3312 {
3313 unsigned int val;
3314
3315 if (optlen != sizeof(val))
3316 return -EINVAL;
3317 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3318 return -EBUSY;
3319 if (copy_from_user(&val, optval, sizeof(val)))
3320 return -EFAULT;
3321 po->tp_loss = !!val;
3322 return 0;
3323 }
8dc41944
HX		 3324 case PACKET_AUXDATA:
3325 {
3326 int val;
3327
3328 if (optlen < sizeof(val))
3329 return -EINVAL;
3330 if (copy_from_user(&val, optval, sizeof(val)))
3331 return -EFAULT;
3332
3333 po->auxdata = !!val;
3334 return 0;
3335 }
80feaacb
PWJ		 3336 case PACKET_ORIGDEV:
3337 {
3338 int val;
3339
3340 if (optlen < sizeof(val))
3341 return -EINVAL;
3342 if (copy_from_user(&val, optval, sizeof(val)))
3343 return -EFAULT;
3344
3345 po->origdev = !!val;
3346 return 0;
3347 }
bfd5f4a3
SS		 3348 case PACKET_VNET_HDR:
3349 {
3350 int val;
3351
3352 if (sock->type != SOCK_RAW)
3353 return -EINVAL;
3354 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3355 return -EBUSY;
3356 if (optlen < sizeof(val))
3357 return -EINVAL;
3358 if (copy_from_user(&val, optval, sizeof(val)))
3359 return -EFAULT;
3360
3361 po->has_vnet_hdr = !!val;
3362 return 0;
3363 }
614f60fa
SM		 3364 case PACKET_TIMESTAMP:
3365 {
3366 int val;
3367
3368 if (optlen != sizeof(val))
3369 return -EINVAL;
3370 if (copy_from_user(&val, optval, sizeof(val)))
3371 return -EFAULT;
3372
3373 po->tp_tstamp = val;
3374 return 0;
3375 }
dc99f600
DM		 3376 case PACKET_FANOUT:
3377 {
3378 int val;
3379
3380 if (optlen != sizeof(val))
3381 return -EINVAL;
3382 if (copy_from_user(&val, optval, sizeof(val)))
3383 return -EFAULT;
3384
3385 return fanout_add(sk, val & 0xffff, val >> 16);
3386 }
5920cd3a
PC		 3387 case PACKET_TX_HAS_OFF:
3388 {
3389 unsigned int val;
3390
3391 if (optlen != sizeof(val))
3392 return -EINVAL;
3393 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3394 return -EBUSY;
3395 if (copy_from_user(&val, optval, sizeof(val)))
3396 return -EFAULT;
3397 po->tp_tx_has_off = !!val;
3398 return 0;
3399 }
d346a3fa
DB		 3400 case PACKET_QDISC_BYPASS:
3401 {
3402 int val;
3403
3404 if (optlen != sizeof(val))
3405 return -EINVAL;
3406 if (copy_from_user(&val, optval, sizeof(val)))
3407 return -EFAULT;
3408
3409 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3410 return 0;
3411 }
1da177e4
LT		 3412 default:
3413 return -ENOPROTOOPT;
3414 }
3415}
3416
3417static int packet_getsockopt(struct socket *sock, int level, int optname,
3418 char __user *optval, int __user *optlen)
3419{
3420 int len;
c06fff6e 3421 int val, lv = sizeof(val);
1da177e4
LT		 3422 struct sock *sk = sock->sk;
3423 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3424 void *data = &val;
ee80fbf3 3425 union tpacket_stats_u st;
1da177e4
LT		 3426
3427 if (level != SOL_PACKET)
3428 return -ENOPROTOOPT;
3429
8ae55f04
KK		 3430 if (get_user(len, optlen))
3431 return -EFAULT;
1da177e4
LT		 3432
3433 if (len < 0)
3434 return -EINVAL;
1ce4f28b 3435
69e3c75f 3436 switch (optname) {
1da177e4 3437 case PACKET_STATISTICS:
1da177e4 3438 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3439 memcpy(&st, &po->stats, sizeof(st));
3440 memset(&po->stats, 0, sizeof(po->stats));
3441 spin_unlock_bh(&sk->sk_receive_queue.lock);
3442
f6fb8f10 3443 if (po->tp_version == TPACKET_V3) {
c06fff6e 3444 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3445 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3446 data = &st.stats3;
f6fb8f10 3447 } else {
c06fff6e 3448 lv = sizeof(struct tpacket_stats);
8bcdeaff 3449 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3450 data = &st.stats1;
f6fb8f10 3451 }
ee80fbf3 3452
8dc41944
HX		 3453 break;
3454 case PACKET_AUXDATA:
8dc41944 3455 val = po->auxdata;
80feaacb
PWJ		 3456 break;
3457 case PACKET_ORIGDEV:
80feaacb 3458 val = po->origdev;
bfd5f4a3
SS		 3459 break;
3460 case PACKET_VNET_HDR:
bfd5f4a3 3461 val = po->has_vnet_hdr;
1da177e4 3462 break;
bbd6ef87 3463 case PACKET_VERSION:
bbd6ef87 3464 val = po->tp_version;
bbd6ef87
PM		 3465 break;
3466 case PACKET_HDRLEN:
3467 if (len > sizeof(int))
3468 len = sizeof(int);
3469 if (copy_from_user(&val, optval, len))
3470 return -EFAULT;
3471 switch (val) {
3472 case TPACKET_V1:
3473 val = sizeof(struct tpacket_hdr);
3474 break;
3475 case TPACKET_V2:
3476 val = sizeof(struct tpacket2_hdr);
3477 break;
f6fb8f10 3478 case TPACKET_V3:
3479 val = sizeof(struct tpacket3_hdr);
3480 break;
bbd6ef87
PM		 3481 default:
3482 return -EINVAL;
3483 }
bbd6ef87 3484 break;
8913336a 3485 case PACKET_RESERVE:
8913336a 3486 val = po->tp_reserve;
8913336a 3487 break;
69e3c75f 3488 case PACKET_LOSS:
69e3c75f 3489 val = po->tp_loss;
69e3c75f 3490 break;
614f60fa 3491 case PACKET_TIMESTAMP:
614f60fa 3492 val = po->tp_tstamp;
614f60fa 3493 break;
dc99f600 3494 case PACKET_FANOUT:
dc99f600
DM		 3495 val = (po->fanout ?
3496 ((u32)po->fanout->id |
77f65ebd
WB		 3497 ((u32)po->fanout->type << 16) |
3498 ((u32)po->fanout->flags << 24)) :
dc99f600 3499 0);
dc99f600 3500 break;
5920cd3a
PC		 3501 case PACKET_TX_HAS_OFF:
3502 val = po->tp_tx_has_off;
3503 break;
d346a3fa
DB		 3504 case PACKET_QDISC_BYPASS:
3505 val = packet_use_direct_xmit(po);
3506 break;
1da177e4
LT		 3507 default:
3508 return -ENOPROTOOPT;
3509 }
3510
c06fff6e
ED		 3511 if (len > lv)
3512 len = lv;
8ae55f04
KK		 3513 if (put_user(len, optlen))
3514 return -EFAULT;
8dc41944
HX		 3515 if (copy_to_user(optval, data, len))
3516 return -EFAULT;
8ae55f04 3517 return 0;
1da177e4
LT		 3518}
3519
3520
351638e7
JP		 3521static int packet_notifier(struct notifier_block *this,
3522 unsigned long msg, void *ptr)
1da177e4
LT		 3523{
3524 struct sock *sk;
351638e7 3525 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3526 struct net *net = dev_net(dev);
1da177e4 3527
808f5114 3528 rcu_read_lock();
b67bfe0d 3529 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3530 struct packet_sock *po = pkt_sk(sk);
3531
3532 switch (msg) {
3533 case NETDEV_UNREGISTER:
1da177e4
LT		 3534 if (po->mclist)
3535 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3536 /* fallthrough */
3537
1da177e4
LT		 3538 case NETDEV_DOWN:
3539 if (dev->ifindex == po->ifindex) {
3540 spin_lock(&po->bind_lock);
3541 if (po->running) {
ce06b03e 3542 __unregister_prot_hook(sk, false);
1da177e4
LT		 3543 sk->sk_err = ENETDOWN;
3544 if (!sock_flag(sk, SOCK_DEAD))
3545 sk->sk_error_report(sk);
3546 }
3547 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3548 packet_cached_dev_reset(po);
1da177e4 3549 po->ifindex = -1;
160ff18a
BG		 3550 if (po->prot_hook.dev)
3551 dev_put(po->prot_hook.dev);
1da177e4
LT		 3552 po->prot_hook.dev = NULL;
3553 }
3554 spin_unlock(&po->bind_lock);
3555 }
3556 break;
3557 case NETDEV_UP:
808f5114 3558 if (dev->ifindex == po->ifindex) {
3559 spin_lock(&po->bind_lock);
ce06b03e
DM		 3560 if (po->num)
3561 register_prot_hook(sk);
808f5114 3562 spin_unlock(&po->bind_lock);
1da177e4 3563 }
1da177e4
LT		 3564 break;
3565 }
3566 }
808f5114 3567 rcu_read_unlock();
1da177e4
LT		 3568 return NOTIFY_DONE;
3569}
3570
3571
3572static int packet_ioctl(struct socket *sock, unsigned int cmd,
3573 unsigned long arg)
3574{
3575 struct sock *sk = sock->sk;
3576
69e3c75f 3577 switch (cmd) {
40d4e3df
ED		 3578 case SIOCOUTQ:
3579 {
3580 int amount = sk_wmem_alloc_get(sk);
31e6d363 3581
40d4e3df
ED		 3582 return put_user(amount, (int __user *)arg);
3583 }
3584 case SIOCINQ:
3585 {
3586 struct sk_buff *skb;
3587 int amount = 0;
3588
3589 spin_lock_bh(&sk->sk_receive_queue.lock);
3590 skb = skb_peek(&sk->sk_receive_queue);
3591 if (skb)
3592 amount = skb->len;
3593 spin_unlock_bh(&sk->sk_receive_queue.lock);
3594 return put_user(amount, (int __user *)arg);
3595 }
3596 case SIOCGSTAMP:
3597 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3598 case SIOCGSTAMPNS:
3599 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3600
1da177e4 3601#ifdef CONFIG_INET
40d4e3df
ED		 3602 case SIOCADDRT:
3603 case SIOCDELRT:
3604 case SIOCDARP:
3605 case SIOCGARP:
3606 case SIOCSARP:
3607 case SIOCGIFADDR:
3608 case SIOCSIFADDR:
3609 case SIOCGIFBRDADDR:
3610 case SIOCSIFBRDADDR:
3611 case SIOCGIFNETMASK:
3612 case SIOCSIFNETMASK:
3613 case SIOCGIFDSTADDR:
3614 case SIOCSIFDSTADDR:
3615 case SIOCSIFFLAGS:
40d4e3df 3616 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3617#endif
3618
40d4e3df
ED		 3619 default:
3620 return -ENOIOCTLCMD;
1da177e4
LT		 3621 }
3622 return 0;
3623}
3624
40d4e3df 3625static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3626 poll_table *wait)
3627{
3628 struct sock *sk = sock->sk;
3629 struct packet_sock *po = pkt_sk(sk);
3630 unsigned int mask = datagram_poll(file, sock, wait);
3631
3632 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3633 if (po->rx_ring.pg_vec) {
f6fb8f10 3634 if (!packet_previous_rx_frame(po, &po->rx_ring,
3635 TP_STATUS_KERNEL))
1da177e4
LT		 3636 mask |= POLLIN | POLLRDNORM;
3637 }
3638 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3639 spin_lock_bh(&sk->sk_write_queue.lock);
3640 if (po->tx_ring.pg_vec) {
3641 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3642 mask |= POLLOUT | POLLWRNORM;
3643 }
3644 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3645 return mask;
3646}
3647
3648
3649/* Dirty? Well, I still did not learn better way to account
3650 * for user mmaps.
3651 */
3652
3653static void packet_mm_open(struct vm_area_struct *vma)
3654{
3655 struct file *file = vma->vm_file;
40d4e3df 3656 struct socket *sock = file->private_data;
1da177e4 3657 struct sock *sk = sock->sk;
1ce4f28b 3658
1da177e4
LT		 3659 if (sk)
3660 atomic_inc(&pkt_sk(sk)->mapped);
3661}
3662
3663static void packet_mm_close(struct vm_area_struct *vma)
3664{
3665 struct file *file = vma->vm_file;
40d4e3df 3666 struct socket *sock = file->private_data;
1da177e4 3667 struct sock *sk = sock->sk;
1ce4f28b 3668
1da177e4
LT		 3669 if (sk)
3670 atomic_dec(&pkt_sk(sk)->mapped);
3671}
3672
f0f37e2f 3673static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3674 .open = packet_mm_open,
3675 .close = packet_mm_close,
1da177e4
LT		 3676};
3677
0e3125c7
NH		 3678static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3679 unsigned int len)
1da177e4
LT		 3680{
3681 int i;
3682
4ebf0ae2 3683 for (i = 0; i < len; i++) {
0e3125c7 3684 if (likely(pg_vec[i].buffer)) {
c56b4d90 3685 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3686 vfree(pg_vec[i].buffer);
3687 else
3688 free_pages((unsigned long)pg_vec[i].buffer,
3689 order);
3690 pg_vec[i].buffer = NULL;
3691 }
1da177e4
LT		 3692 }
3693 kfree(pg_vec);
3694}
3695
eea49cc9 3696static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3697{
f0d4eb29 3698 char *buffer;
0e3125c7
NH		 3699 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3700 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3701
3702 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3703 if (buffer)
3704 return buffer;
3705
f0d4eb29 3706 /* __get_free_pages failed, fall back to vmalloc */
bbce5a59 3707 buffer = vzalloc((1 << order) * PAGE_SIZE);
0e3125c7
NH		 3708 if (buffer)
3709 return buffer;
3710
f0d4eb29 3711 /* vmalloc failed, lets dig into swap here */
0e3125c7 3712 gfp_flags &= ~__GFP_NORETRY;
f0d4eb29 3713 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3714 if (buffer)
3715 return buffer;
3716
f0d4eb29 3717 /* complete and utter failure */
0e3125c7 3718 return NULL;
4ebf0ae2
DM		 3719}
3720
0e3125c7 3721static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3722{
3723 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3724 struct pgv *pg_vec;
4ebf0ae2
DM		 3725 int i;
3726
0e3125c7 3727 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3728 if (unlikely(!pg_vec))
3729 goto out;
3730
3731 for (i = 0; i < block_nr; i++) {
c56b4d90 3732 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3733 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3734 goto out_free_pgvec;
3735 }
3736
3737out:
3738 return pg_vec;
3739
3740out_free_pgvec:
3741 free_pg_vec(pg_vec, order, block_nr);
3742 pg_vec = NULL;
3743 goto out;
3744}
1da177e4 3745
f6fb8f10 3746static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3747 int closing, int tx_ring)
1da177e4 3748{
0e3125c7 3749 struct pgv *pg_vec = NULL;
1da177e4 3750 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3751 int was_running, order = 0;
69e3c75f
JB		 3752 struct packet_ring_buffer *rb;
3753 struct sk_buff_head *rb_queue;
0e11c91e 3754 __be16 num;
f6fb8f10 3755 int err = -EINVAL;
3756 /* Added to avoid minimal code churn */
3757 struct tpacket_req *req = &req_u->req;
3758
3759 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3760 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3761 WARN(1, "Tx-ring is not supported.\n");
3762 goto out;
3763 }
1ce4f28b 3764
69e3c75f
JB		 3765 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3766 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3767
69e3c75f
JB		 3768 err = -EBUSY;
3769 if (!closing) {
3770 if (atomic_read(&po->mapped))
3771 goto out;
b0138408 3772 if (packet_read_pending(rb))
69e3c75f
JB		 3773 goto out;
3774 }
1da177e4 3775
69e3c75f
JB		 3776 if (req->tp_block_nr) {
3777 /* Sanity tests and some calculations */
3778 err = -EBUSY;
3779 if (unlikely(rb->pg_vec))
3780 goto out;
1da177e4 3781
bbd6ef87
PM		 3782 switch (po->tp_version) {
3783 case TPACKET_V1:
3784 po->tp_hdrlen = TPACKET_HDRLEN;
3785 break;
3786 case TPACKET_V2:
3787 po->tp_hdrlen = TPACKET2_HDRLEN;
3788 break;
f6fb8f10 3789 case TPACKET_V3:
3790 po->tp_hdrlen = TPACKET3_HDRLEN;
3791 break;
bbd6ef87
PM		 3792 }
3793
69e3c75f 3794 err = -EINVAL;
4ebf0ae2 3795 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3796 goto out;
4ebf0ae2 3797 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3798 goto out;
dc808110
ED		 3799 if (po->tp_version >= TPACKET_V3 &&
3800 (int)(req->tp_block_size -
3801 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
3802 goto out;
8913336a 3803 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3804 po->tp_reserve))
3805 goto out;
4ebf0ae2 3806 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3807 goto out;
1da177e4 3808
69e3c75f
JB		 3809 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3810 if (unlikely(rb->frames_per_block <= 0))
3811 goto out;
3812 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3813 req->tp_frame_nr))
3814 goto out;
1da177e4
LT		 3815
3816 err = -ENOMEM;
4ebf0ae2
DM		 3817 order = get_order(req->tp_block_size);
3818 pg_vec = alloc_pg_vec(req, order);
3819 if (unlikely(!pg_vec))
1da177e4 3820 goto out;
f6fb8f10 3821 switch (po->tp_version) {
3822 case TPACKET_V3:
3823 /* Transmit path is not supported. We checked
3824 * it above but just being paranoid
3825 */
3826 if (!tx_ring)
3827 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
d7cf0c34 3828 break;
f6fb8f10 3829 default:
3830 break;
3831 }
69e3c75f
JB		 3832 }
3833 /* Done */
3834 else {
3835 err = -EINVAL;
4ebf0ae2 3836 if (unlikely(req->tp_frame_nr))
69e3c75f 3837 goto out;
1da177e4
LT		 3838 }
3839
3840 lock_sock(sk);
3841
3842 /* Detach socket from network */
3843 spin_lock(&po->bind_lock);
3844 was_running = po->running;
3845 num = po->num;
3846 if (was_running) {
1da177e4 3847 po->num = 0;
ce06b03e 3848 __unregister_prot_hook(sk, false);
1da177e4
LT		 3849 }
3850 spin_unlock(&po->bind_lock);
1ce4f28b 3851
1da177e4
LT		 3852 synchronize_net();
3853
3854 err = -EBUSY;
905db440 3855 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3856 if (closing || atomic_read(&po->mapped) == 0) {
3857 err = 0;
69e3c75f 3858 spin_lock_bh(&rb_queue->lock);
c053fd96 3859 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3860 rb->frame_max = (req->tp_frame_nr - 1);
3861 rb->head = 0;
3862 rb->frame_size = req->tp_frame_size;
3863 spin_unlock_bh(&rb_queue->lock);
3864
c053fd96
CG		 3865 swap(rb->pg_vec_order, order);
3866 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3867
3868 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3869 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3870 tpacket_rcv : packet_rcv;
3871 skb_queue_purge(rb_queue);
1da177e4 3872 if (atomic_read(&po->mapped))
40d4e3df
ED		 3873 pr_err("packet_mmap: vma is busy: %d\n",
3874 atomic_read(&po->mapped));
1da177e4 3875 }
905db440 3876 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3877
3878 spin_lock(&po->bind_lock);
ce06b03e 3879 if (was_running) {
1da177e4 3880 po->num = num;
ce06b03e 3881 register_prot_hook(sk);
1da177e4
LT		 3882 }
3883 spin_unlock(&po->bind_lock);
f6fb8f10 3884 if (closing && (po->tp_version > TPACKET_V2)) {
3885 /* Because we don't support block-based V3 on tx-ring */
3886 if (!tx_ring)
3887 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3888 }
1da177e4
LT		 3889 release_sock(sk);
3890
1da177e4
LT		 3891 if (pg_vec)
3892 free_pg_vec(pg_vec, order, req->tp_block_nr);
3893out:
3894 return err;
3895}
3896
69e3c75f
JB		 3897static int packet_mmap(struct file *file, struct socket *sock,
3898 struct vm_area_struct *vma)
1da177e4
LT		 3899{
3900 struct sock *sk = sock->sk;
3901 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3902 unsigned long size, expected_size;
3903 struct packet_ring_buffer *rb;
1da177e4
LT		 3904 unsigned long start;
3905 int err = -EINVAL;
3906 int i;
3907
3908 if (vma->vm_pgoff)
3909 return -EINVAL;
3910
905db440 3911 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3912
3913 expected_size = 0;
3914 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3915 if (rb->pg_vec) {
3916 expected_size += rb->pg_vec_len
3917 * rb->pg_vec_pages
3918 * PAGE_SIZE;
3919 }
3920 }
3921
3922 if (expected_size == 0)
1da177e4 3923 goto out;
69e3c75f
JB		 3924
3925 size = vma->vm_end - vma->vm_start;
3926 if (size != expected_size)
1da177e4
LT		 3927 goto out;
3928
1da177e4 3929 start = vma->vm_start;
69e3c75f
JB		 3930 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3931 if (rb->pg_vec == NULL)
3932 continue;
3933
3934 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3935 struct page *page;
3936 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3937 int pg_num;
3938
c56b4d90
CG		 3939 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3940 page = pgv_to_page(kaddr);
69e3c75f
JB		 3941 err = vm_insert_page(vma, start, page);
3942 if (unlikely(err))
3943 goto out;
3944 start += PAGE_SIZE;
0e3125c7 3945 kaddr += PAGE_SIZE;
69e3c75f 3946 }
4ebf0ae2 3947 }
1da177e4 3948 }
69e3c75f 3949
4ebf0ae2 3950 atomic_inc(&po->mapped);
1da177e4
LT		 3951 vma->vm_ops = &packet_mmap_ops;
3952 err = 0;
3953
3954out:
905db440 3955 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3956 return err;
3957}
1da177e4 3958
90ddc4f0 3959static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3960 .family = PF_PACKET,
3961 .owner = THIS_MODULE,
3962 .release = packet_release,
3963 .bind = packet_bind_spkt,
3964 .connect = sock_no_connect,
3965 .socketpair = sock_no_socketpair,
3966 .accept = sock_no_accept,
3967 .getname = packet_getname_spkt,
3968 .poll = datagram_poll,
3969 .ioctl = packet_ioctl,
3970 .listen = sock_no_listen,
3971 .shutdown = sock_no_shutdown,
3972 .setsockopt = sock_no_setsockopt,
3973 .getsockopt = sock_no_getsockopt,
3974 .sendmsg = packet_sendmsg_spkt,
3975 .recvmsg = packet_recvmsg,
3976 .mmap = sock_no_mmap,
3977 .sendpage = sock_no_sendpage,
3978};
1da177e4 3979
90ddc4f0 3980static const struct proto_ops packet_ops = {
1da177e4
LT		 3981 .family = PF_PACKET,
3982 .owner = THIS_MODULE,
3983 .release = packet_release,
3984 .bind = packet_bind,
3985 .connect = sock_no_connect,
3986 .socketpair = sock_no_socketpair,
3987 .accept = sock_no_accept,
1ce4f28b 3988 .getname = packet_getname,
1da177e4
LT		 3989 .poll = packet_poll,
3990 .ioctl = packet_ioctl,
3991 .listen = sock_no_listen,
3992 .shutdown = sock_no_shutdown,
3993 .setsockopt = packet_setsockopt,
3994 .getsockopt = packet_getsockopt,
3995 .sendmsg = packet_sendmsg,
3996 .recvmsg = packet_recvmsg,
3997 .mmap = packet_mmap,
3998 .sendpage = sock_no_sendpage,
3999};
4000
ec1b4cf7 4001static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 4002 .family = PF_PACKET,
4003 .create = packet_create,
4004 .owner = THIS_MODULE,
4005};
4006
4007static struct notifier_block packet_netdev_notifier = {
40d4e3df 4008 .notifier_call = packet_notifier,
1da177e4
LT		 4009};
4010
4011#ifdef CONFIG_PROC_FS
1da177e4
LT		 4012
4013static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 4014 __acquires(RCU)
1da177e4 4015{
e372c414 4016 struct net *net = seq_file_net(seq);
808f5114 4017
4018 rcu_read_lock();
4019 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 4020}
4021
4022static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4023{
1bf40954 4024 struct net *net = seq_file_net(seq);
808f5114 4025 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 4026}
4027
4028static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 4029 __releases(RCU)
1da177e4 4030{
808f5114 4031 rcu_read_unlock();
1da177e4
LT		 4032}
4033
1ce4f28b 4034static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 4035{
4036 if (v == SEQ_START_TOKEN)
4037 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4038 else {
b7ceabd9 4039 struct sock *s = sk_entry(v);
1da177e4
LT		 4040 const struct packet_sock *po = pkt_sk(s);
4041
4042 seq_printf(seq,
71338aa7 4043 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 4044 s,
4045 atomic_read(&s->sk_refcnt),
4046 s->sk_type,
4047 ntohs(po->num),
4048 po->ifindex,
4049 po->running,
4050 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4051 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4052 sock_i_ino(s));
1da177e4
LT		 4053 }
4054
4055 return 0;
4056}
4057
56b3d975 4058static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4059 .start = packet_seq_start,
4060 .next = packet_seq_next,
4061 .stop = packet_seq_stop,
4062 .show = packet_seq_show,
4063};
4064
4065static int packet_seq_open(struct inode *inode, struct file *file)
4066{
e372c414
DL		 4067 return seq_open_net(inode, file, &packet_seq_ops,
4068 sizeof(struct seq_net_private));
1da177e4
LT		 4069}
4070
da7071d7 4071static const struct file_operations packet_seq_fops = {
1da177e4
LT		 4072 .owner = THIS_MODULE,
4073 .open = packet_seq_open,
4074 .read = seq_read,
4075 .llseek = seq_lseek,
e372c414 4076 .release = seq_release_net,
1da177e4
LT		 4077};
4078
4079#endif
4080
2c8c1e72 4081static int __net_init packet_net_init(struct net *net)
d12d01d6 4082{
0fa7fa98 4083 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4084 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4085
d4beaa66 4086 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 4087 return -ENOMEM;
4088
4089 return 0;
4090}
4091
2c8c1e72 4092static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4093{
ece31ffd 4094 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 4095}
4096
4097static struct pernet_operations packet_net_ops = {
4098 .init = packet_net_init,
4099 .exit = packet_net_exit,
4100};
4101
4102
1da177e4
LT		 4103static void __exit packet_exit(void)
4104{
1da177e4 4105 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4106 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4107 sock_unregister(PF_PACKET);
4108 proto_unregister(&packet_proto);
4109}
4110
4111static int __init packet_init(void)
4112{
4113 int rc = proto_register(&packet_proto, 0);
4114
4115 if (rc != 0)
4116 goto out;
4117
4118 sock_register(&packet_family_ops);
d12d01d6 4119 register_pernet_subsys(&packet_net_ops);
1da177e4 4120 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4121out:
4122 return rc;
4123}
4124
4125module_init(packet_init);
4126module_exit(packet_exit);
4127MODULE_LICENSE("GPL");
4128MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 1.173912 seconds and 5 git commands to generate.