projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
packet: fill the gap of TPACKET_ALIGNMENT with zeros
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
5df0ddfb 91#include <linux/reciprocal_div.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
d346a3fa
DB		 240static int packet_direct_xmit(struct sk_buff *skb)
241{
242 struct net_device *dev = skb->dev;
243 const struct net_device_ops *ops = dev->netdev_ops;
244 netdev_features_t features;
245 struct netdev_queue *txq;
246 u16 queue_map;
247 int ret;
248
249 if (unlikely(!netif_running(dev) ||
250 !netif_carrier_ok(dev))) {
251 kfree_skb(skb);
252 return NET_XMIT_DROP;
253 }
254
255 features = netif_skb_features(skb);
256 if (skb_needs_linearize(skb, features) &&
257 __skb_linearize(skb)) {
258 kfree_skb(skb);
259 return NET_XMIT_DROP;
260 }
261
262 queue_map = skb_get_queue_mapping(skb);
263 txq = netdev_get_tx_queue(dev, queue_map);
264
265 __netif_tx_lock_bh(txq);
266 if (unlikely(netif_xmit_frozen_or_stopped(txq))) {
267 ret = NETDEV_TX_BUSY;
268 kfree_skb(skb);
269 goto out;
270 }
271
272 ret = ops->ndo_start_xmit(skb, dev);
273 if (likely(dev_xmit_complete(ret)))
274 txq_trans_update(txq);
275 else
276 kfree_skb(skb);
277out:
278 __netif_tx_unlock_bh(txq);
279 return ret;
280}
281
66e56cd4
DB		 282static struct net_device *packet_cached_dev_get(struct packet_sock *po)
283{
284 struct net_device *dev;
285
286 rcu_read_lock();
287 dev = rcu_dereference(po->cached_dev);
288 if (likely(dev))
289 dev_hold(dev);
290 rcu_read_unlock();
291
292 return dev;
293}
294
295static void packet_cached_dev_assign(struct packet_sock *po,
296 struct net_device *dev)
297{
298 rcu_assign_pointer(po->cached_dev, dev);
299}
300
301static void packet_cached_dev_reset(struct packet_sock *po)
302{
303 RCU_INIT_POINTER(po->cached_dev, NULL);
304}
305
d346a3fa
DB		 306static bool packet_use_direct_xmit(const struct packet_sock *po)
307{
308 return po->xmit == packet_direct_xmit;
309}
310
311static u16 packet_pick_tx_queue(struct net_device *dev)
312{
1cbac010 313 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 314}
315
ce06b03e
DM		 316/* register_prot_hook must be invoked with the po->bind_lock held,
317 * or from a context in which asynchronous accesses to the packet
318 * socket is not possible (packet_create()).
319 */
320static void register_prot_hook(struct sock *sk)
321{
322 struct packet_sock *po = pkt_sk(sk);
e40526cb 323
ce06b03e 324 if (!po->running) {
66e56cd4 325 if (po->fanout)
dc99f600 326 __fanout_link(sk, po);
66e56cd4 327 else
dc99f600 328 dev_add_pack(&po->prot_hook);
e40526cb 329
ce06b03e
DM		 330 sock_hold(sk);
331 po->running = 1;
332 }
333}
334
335/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
336 * held. If the sync parameter is true, we will temporarily drop
337 * the po->bind_lock and do a synchronize_net to make sure no
338 * asynchronous packet processing paths still refer to the elements
339 * of po->prot_hook. If the sync parameter is false, it is the
340 * callers responsibility to take care of this.
341 */
342static void __unregister_prot_hook(struct sock *sk, bool sync)
343{
344 struct packet_sock *po = pkt_sk(sk);
345
346 po->running = 0;
66e56cd4
DB		 347
348 if (po->fanout)
dc99f600 349 __fanout_unlink(sk, po);
66e56cd4 350 else
dc99f600 351 __dev_remove_pack(&po->prot_hook);
e40526cb 352
ce06b03e
DM		 353 __sock_put(sk);
354
355 if (sync) {
356 spin_unlock(&po->bind_lock);
357 synchronize_net();
358 spin_lock(&po->bind_lock);
359 }
360}
361
362static void unregister_prot_hook(struct sock *sk, bool sync)
363{
364 struct packet_sock *po = pkt_sk(sk);
365
366 if (po->running)
367 __unregister_prot_hook(sk, sync);
368}
369
f6dafa95 370static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 371{
372 if (is_vmalloc_addr(addr))
373 return vmalloc_to_page(addr);
374 return virt_to_page(addr);
375}
376
69e3c75f 377static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 378{
184f489e 379 union tpacket_uhdr h;
1da177e4 380
69e3c75f 381 h.raw = frame;
bbd6ef87
PM		 382 switch (po->tp_version) {
383 case TPACKET_V1:
69e3c75f 384 h.h1->tp_status = status;
0af55bb5 385 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 386 break;
387 case TPACKET_V2:
69e3c75f 388 h.h2->tp_status = status;
0af55bb5 389 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 390 break;
f6fb8f10 391 case TPACKET_V3:
69e3c75f 392 default:
f6fb8f10 393 WARN(1, "TPACKET version not supported.\n");
69e3c75f 394 BUG();
bbd6ef87 395 }
69e3c75f
JB		 396
397 smp_wmb();
bbd6ef87
PM		 398}
399
69e3c75f 400static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 401{
184f489e 402 union tpacket_uhdr h;
bbd6ef87 403
69e3c75f
JB		 404 smp_rmb();
405
bbd6ef87
PM		 406 h.raw = frame;
407 switch (po->tp_version) {
408 case TPACKET_V1:
0af55bb5 409 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 410 return h.h1->tp_status;
bbd6ef87 411 case TPACKET_V2:
0af55bb5 412 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 413 return h.h2->tp_status;
f6fb8f10 414 case TPACKET_V3:
69e3c75f 415 default:
f6fb8f10 416 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 417 BUG();
418 return 0;
bbd6ef87 419 }
1da177e4 420}
69e3c75f 421
b9c32fb2
DB		 422static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
423 unsigned int flags)
7a51384c
DB		 424{
425 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
426
427 if (shhwtstamps) {
428 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
429 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
b9c32fb2 430 return TP_STATUS_TS_SYS_HARDWARE;
7a51384c
DB		 431 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
432 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
b9c32fb2 433 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 434 }
435
436 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 437 return TP_STATUS_TS_SOFTWARE;
7a51384c 438
b9c32fb2 439 return 0;
7a51384c
DB		 440}
441
b9c32fb2
DB		 442static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
443 struct sk_buff *skb)
2e31396f
WB		 444{
445 union tpacket_uhdr h;
446 struct timespec ts;
b9c32fb2 447 __u32 ts_status;
2e31396f 448
b9c32fb2
DB		 449 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
450 return 0;
2e31396f
WB		 451
452 h.raw = frame;
453 switch (po->tp_version) {
454 case TPACKET_V1:
455 h.h1->tp_sec = ts.tv_sec;
456 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
457 break;
458 case TPACKET_V2:
459 h.h2->tp_sec = ts.tv_sec;
460 h.h2->tp_nsec = ts.tv_nsec;
461 break;
462 case TPACKET_V3:
463 default:
464 WARN(1, "TPACKET version not supported.\n");
465 BUG();
466 }
467
468 /* one flush is safe, as both fields always lie on the same cacheline */
469 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
470 smp_wmb();
b9c32fb2
DB		 471
472 return ts_status;
2e31396f
WB		 473}
474
69e3c75f
JB		 475static void *packet_lookup_frame(struct packet_sock *po,
476 struct packet_ring_buffer *rb,
477 unsigned int position,
478 int status)
479{
480 unsigned int pg_vec_pos, frame_offset;
184f489e 481 union tpacket_uhdr h;
69e3c75f
JB		 482
483 pg_vec_pos = position / rb->frames_per_block;
484 frame_offset = position % rb->frames_per_block;
485
0e3125c7
NH		 486 h.raw = rb->pg_vec[pg_vec_pos].buffer +
487 (frame_offset * rb->frame_size);
69e3c75f
JB		 488
489 if (status != __packet_get_status(po, h.raw))
490 return NULL;
491
492 return h.raw;
493}
494
eea49cc9 495static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 496 struct packet_ring_buffer *rb,
497 int status)
498{
499 return packet_lookup_frame(po, rb, rb->head, status);
500}
501
bc59ba39 502static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 503{
504 del_timer_sync(&pkc->retire_blk_timer);
505}
506
507static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
508 int tx_ring,
509 struct sk_buff_head *rb_queue)
510{
bc59ba39 511 struct tpacket_kbdq_core *pkc;
f6fb8f10 512
22781a5b
DJ		 513 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
514 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 515
ec6f809f 516 spin_lock_bh(&rb_queue->lock);
f6fb8f10 517 pkc->delete_blk_timer = 1;
ec6f809f 518 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 519
520 prb_del_retire_blk_timer(pkc);
521}
522
523static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 524 struct tpacket_kbdq_core *pkc,
f6fb8f10 525 void (*func) (unsigned long))
526{
527 init_timer(&pkc->retire_blk_timer);
528 pkc->retire_blk_timer.data = (long)po;
529 pkc->retire_blk_timer.function = func;
530 pkc->retire_blk_timer.expires = jiffies;
531}
532
533static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
534{
bc59ba39 535 struct tpacket_kbdq_core *pkc;
f6fb8f10 536
537 if (tx_ring)
538 BUG();
539
22781a5b
DJ		 540 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
541 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 542 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
543}
544
545static int prb_calc_retire_blk_tmo(struct packet_sock *po,
546 int blk_size_in_bytes)
547{
548 struct net_device *dev;
549 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 550 struct ethtool_cmd ecmd;
551 int err;
e440cf2c 552 u32 speed;
f6fb8f10 553
4bc71cb9
JP		 554 rtnl_lock();
555 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
556 if (unlikely(!dev)) {
557 rtnl_unlock();
f6fb8f10 558 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 559 }
560 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 561 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 562 rtnl_unlock();
563 if (!err) {
4bc71cb9
JP		 564 /*
565 * If the link speed is so slow you don't really
566 * need to worry about perf anyways
567 */
e440cf2c 568 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 569 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 570 } else {
571 msec = 1;
572 div = speed / 1000;
f6fb8f10 573 }
574 }
575
576 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
577
578 if (div)
579 mbits /= div;
580
581 tmo = mbits * msec;
582
583 if (div)
584 return tmo+1;
585 return tmo;
586}
587
bc59ba39 588static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 589 union tpacket_req_u *req_u)
590{
591 p1->feature_req_word = req_u->req3.tp_feature_req_word;
592}
593
594static void init_prb_bdqc(struct packet_sock *po,
595 struct packet_ring_buffer *rb,
596 struct pgv *pg_vec,
597 union tpacket_req_u *req_u, int tx_ring)
598{
22781a5b 599 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 600 struct tpacket_block_desc *pbd;
f6fb8f10 601
602 memset(p1, 0x0, sizeof(*p1));
603
604 p1->knxt_seq_num = 1;
605 p1->pkbdq = pg_vec;
bc59ba39 606 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 607 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 608 p1->kblk_size = req_u->req3.tp_block_size;
609 p1->knum_blocks = req_u->req3.tp_block_nr;
610 p1->hdrlen = po->tp_hdrlen;
611 p1->version = po->tp_version;
612 p1->last_kactive_blk_num = 0;
ee80fbf3 613 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 614 if (req_u->req3.tp_retire_blk_tov)
615 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
616 else
617 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
618 req_u->req3.tp_block_size);
619 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
620 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
621
622 prb_init_ft_ops(p1, req_u);
623 prb_setup_retire_blk_timer(po, tx_ring);
624 prb_open_block(p1, pbd);
625}
626
627/* Do NOT update the last_blk_num first.
628 * Assumes sk_buff_head lock is held.
629 */
bc59ba39 630static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 631{
632 mod_timer(&pkc->retire_blk_timer,
633 jiffies + pkc->tov_in_jiffies);
634 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
635}
636
637/*
638 * Timer logic:
639 * 1) We refresh the timer only when we open a block.
640 * By doing this we don't waste cycles refreshing the timer
641 * on packet-by-packet basis.
642 *
643 * With a 1MB block-size, on a 1Gbps line, it will take
644 * i) ~8 ms to fill a block + ii) memcpy etc.
645 * In this cut we are not accounting for the memcpy time.
646 *
647 * So, if the user sets the 'tmo' to 10ms then the timer
648 * will never fire while the block is still getting filled
649 * (which is what we want). However, the user could choose
650 * to close a block early and that's fine.
651 *
652 * But when the timer does fire, we check whether or not to refresh it.
653 * Since the tmo granularity is in msecs, it is not too expensive
654 * to refresh the timer, lets say every '8' msecs.
655 * Either the user can set the 'tmo' or we can derive it based on
656 * a) line-speed and b) block-size.
657 * prb_calc_retire_blk_tmo() calculates the tmo.
658 *
659 */
660static void prb_retire_rx_blk_timer_expired(unsigned long data)
661{
662 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 663 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 664 unsigned int frozen;
bc59ba39 665 struct tpacket_block_desc *pbd;
f6fb8f10 666
667 spin_lock(&po->sk.sk_receive_queue.lock);
668
669 frozen = prb_queue_frozen(pkc);
670 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
671
672 if (unlikely(pkc->delete_blk_timer))
673 goto out;
674
675 /* We only need to plug the race when the block is partially filled.
676 * tpacket_rcv:
677 * lock(); increment BLOCK_NUM_PKTS; unlock()
678 * copy_bits() is in progress ...
679 * timer fires on other cpu:
680 * we can't retire the current block because copy_bits
681 * is in progress.
682 *
683 */
684 if (BLOCK_NUM_PKTS(pbd)) {
685 while (atomic_read(&pkc->blk_fill_in_prog)) {
686 /* Waiting for skb_copy_bits to finish... */
687 cpu_relax();
688 }
689 }
690
691 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
692 if (!frozen) {
693 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
694 if (!prb_dispatch_next_block(pkc, po))
695 goto refresh_timer;
696 else
697 goto out;
698 } else {
699 /* Case 1. Queue was frozen because user-space was
700 * lagging behind.
701 */
702 if (prb_curr_blk_in_use(pkc, pbd)) {
703 /*
704 * Ok, user-space is still behind.
705 * So just refresh the timer.
706 */
707 goto refresh_timer;
708 } else {
709 /* Case 2. queue was frozen,user-space caught up,
710 * now the link went idle && the timer fired.
711 * We don't have a block to close.So we open this
712 * block and restart the timer.
713 * opening a block thaws the queue,restarts timer
714 * Thawing/timer-refresh is a side effect.
715 */
716 prb_open_block(pkc, pbd);
717 goto out;
718 }
719 }
720 }
721
722refresh_timer:
723 _prb_refresh_rx_retire_blk_timer(pkc);
724
725out:
726 spin_unlock(&po->sk.sk_receive_queue.lock);
727}
728
eea49cc9 729static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 730 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 731{
732 /* Flush everything minus the block header */
733
734#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
735 u8 *start, *end;
736
737 start = (u8 *)pbd1;
738
739 /* Skip the block header(we know header WILL fit in 4K) */
740 start += PAGE_SIZE;
741
742 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
743 for (; start < end; start += PAGE_SIZE)
744 flush_dcache_page(pgv_to_page(start));
745
746 smp_wmb();
747#endif
748
749 /* Now update the block status. */
750
751 BLOCK_STATUS(pbd1) = status;
752
753 /* Flush the block header */
754
755#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
756 start = (u8 *)pbd1;
757 flush_dcache_page(pgv_to_page(start));
758
759 smp_wmb();
760#endif
761}
762
763/*
764 * Side effect:
765 *
766 * 1) flush the block
767 * 2) Increment active_blk_num
768 *
769 * Note:We DONT refresh the timer on purpose.
770 * Because almost always the next block will be opened.
771 */
bc59ba39 772static void prb_close_block(struct tpacket_kbdq_core *pkc1,
773 struct tpacket_block_desc *pbd1,
f6fb8f10 774 struct packet_sock *po, unsigned int stat)
775{
776 __u32 status = TP_STATUS_USER | stat;
777
778 struct tpacket3_hdr *last_pkt;
bc59ba39 779 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 780
ee80fbf3 781 if (po->stats.stats3.tp_drops)
f6fb8f10 782 status |= TP_STATUS_LOSING;
783
784 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
785 last_pkt->tp_next_offset = 0;
786
787 /* Get the ts of the last pkt */
788 if (BLOCK_NUM_PKTS(pbd1)) {
789 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
790 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
791 } else {
792 /* Ok, we tmo'd - so get the current time */
793 struct timespec ts;
794 getnstimeofday(&ts);
795 h1->ts_last_pkt.ts_sec = ts.tv_sec;
796 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
797 }
798
799 smp_wmb();
800
801 /* Flush the block */
802 prb_flush_block(pkc1, pbd1, status);
803
804 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
805}
806
eea49cc9 807static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 808{
809 pkc->reset_pending_on_curr_blk = 0;
810}
811
812/*
813 * Side effect of opening a block:
814 *
815 * 1) prb_queue is thawed.
816 * 2) retire_blk_timer is refreshed.
817 *
818 */
bc59ba39 819static void prb_open_block(struct tpacket_kbdq_core *pkc1,
820 struct tpacket_block_desc *pbd1)
f6fb8f10 821{
822 struct timespec ts;
bc59ba39 823 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 824
825 smp_rmb();
826
8da3056c
DB		 827 /* We could have just memset this but we will lose the
828 * flexibility of making the priv area sticky
829 */
f6fb8f10 830
8da3056c
DB		 831 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
832 BLOCK_NUM_PKTS(pbd1) = 0;
833 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 834
8da3056c
DB		 835 getnstimeofday(&ts);
836
837 h1->ts_first_pkt.ts_sec = ts.tv_sec;
838 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 839
8da3056c
DB		 840 pkc1->pkblk_start = (char *)pbd1;
841 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
842
843 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
844 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
845
846 pbd1->version = pkc1->version;
847 pkc1->prev = pkc1->nxt_offset;
848 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
849
850 prb_thaw_queue(pkc1);
851 _prb_refresh_rx_retire_blk_timer(pkc1);
852
853 smp_wmb();
f6fb8f10 854}
855
856/*
857 * Queue freeze logic:
858 * 1) Assume tp_block_nr = 8 blocks.
859 * 2) At time 't0', user opens Rx ring.
860 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
861 * 4) user-space is either sleeping or processing block '0'.
862 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
863 * it will close block-7,loop around and try to fill block '0'.
864 * call-flow:
865 * __packet_lookup_frame_in_block
866 * prb_retire_current_block()
867 * prb_dispatch_next_block()
868 * |->(BLOCK_STATUS == USER) evaluates to true
869 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
870 * 6) Now there are two cases:
871 * 6.1) Link goes idle right after the queue is frozen.
872 * But remember, the last open_block() refreshed the timer.
873 * When this timer expires,it will refresh itself so that we can
874 * re-open block-0 in near future.
875 * 6.2) Link is busy and keeps on receiving packets. This is a simple
876 * case and __packet_lookup_frame_in_block will check if block-0
877 * is free and can now be re-used.
878 */
eea49cc9 879static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 880 struct packet_sock *po)
881{
882 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 883 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 884}
885
886#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
887
888/*
889 * If the next block is free then we will dispatch it
890 * and return a good offset.
891 * Else, we will freeze the queue.
892 * So, caller must check the return value.
893 */
bc59ba39 894static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 895 struct packet_sock *po)
896{
bc59ba39 897 struct tpacket_block_desc *pbd;
f6fb8f10 898
899 smp_rmb();
900
901 /* 1. Get current block num */
902 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
903
904 /* 2. If this block is currently in_use then freeze the queue */
905 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
906 prb_freeze_queue(pkc, po);
907 return NULL;
908 }
909
910 /*
911 * 3.
912 * open this block and return the offset where the first packet
913 * needs to get stored.
914 */
915 prb_open_block(pkc, pbd);
916 return (void *)pkc->nxt_offset;
917}
918
bc59ba39 919static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 920 struct packet_sock *po, unsigned int status)
921{
bc59ba39 922 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 923
924 /* retire/close the current block */
925 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
926 /*
927 * Plug the case where copy_bits() is in progress on
928 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
929 * have space to copy the pkt in the current block and
930 * called prb_retire_current_block()
931 *
932 * We don't need to worry about the TMO case because
933 * the timer-handler already handled this case.
934 */
935 if (!(status & TP_STATUS_BLK_TMO)) {
936 while (atomic_read(&pkc->blk_fill_in_prog)) {
937 /* Waiting for skb_copy_bits to finish... */
938 cpu_relax();
939 }
940 }
941 prb_close_block(pkc, pbd, po, status);
942 return;
943 }
f6fb8f10 944}
945
eea49cc9 946static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 947 struct tpacket_block_desc *pbd)
f6fb8f10 948{
949 return TP_STATUS_USER & BLOCK_STATUS(pbd);
950}
951
eea49cc9 952static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 953{
954 return pkc->reset_pending_on_curr_blk;
955}
956
eea49cc9 957static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 958{
bc59ba39 959 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 960 atomic_dec(&pkc->blk_fill_in_prog);
961}
962
eea49cc9 963static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 964 struct tpacket3_hdr *ppd)
965{
3958afa1 966 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 967}
968
eea49cc9 969static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 970 struct tpacket3_hdr *ppd)
971{
972 ppd->hv1.tp_rxhash = 0;
973}
974
eea49cc9 975static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 976 struct tpacket3_hdr *ppd)
977{
978 if (vlan_tx_tag_present(pkc->skb)) {
979 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
980 ppd->tp_status = TP_STATUS_VLAN_VALID;
981 } else {
9e67030a 982 ppd->hv1.tp_vlan_tci = 0;
983 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 984 }
985}
986
bc59ba39 987static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 988 struct tpacket3_hdr *ppd)
989{
990 prb_fill_vlan_info(pkc, ppd);
991
992 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
993 prb_fill_rxhash(pkc, ppd);
994 else
995 prb_clear_rxhash(pkc, ppd);
996}
997
eea49cc9 998static void prb_fill_curr_block(char *curr,
bc59ba39 999 struct tpacket_kbdq_core *pkc,
1000 struct tpacket_block_desc *pbd,
f6fb8f10 1001 unsigned int len)
1002{
1003 struct tpacket3_hdr *ppd;
1004
1005 ppd = (struct tpacket3_hdr *)curr;
1006 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1007 pkc->prev = curr;
1008 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1009 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1010 BLOCK_NUM_PKTS(pbd) += 1;
1011 atomic_inc(&pkc->blk_fill_in_prog);
1012 prb_run_all_ft_ops(pkc, ppd);
1013}
1014
1015/* Assumes caller has the sk->rx_queue.lock */
1016static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1017 struct sk_buff *skb,
1018 int status,
1019 unsigned int len
1020 )
1021{
bc59ba39 1022 struct tpacket_kbdq_core *pkc;
1023 struct tpacket_block_desc *pbd;
f6fb8f10 1024 char *curr, *end;
1025
e3192690 1026 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1027 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1028
1029 /* Queue is frozen when user space is lagging behind */
1030 if (prb_queue_frozen(pkc)) {
1031 /*
1032 * Check if that last block which caused the queue to freeze,
1033 * is still in_use by user-space.
1034 */
1035 if (prb_curr_blk_in_use(pkc, pbd)) {
1036 /* Can't record this packet */
1037 return NULL;
1038 } else {
1039 /*
1040 * Ok, the block was released by user-space.
1041 * Now let's open that block.
1042 * opening a block also thaws the queue.
1043 * Thawing is a side effect.
1044 */
1045 prb_open_block(pkc, pbd);
1046 }
1047 }
1048
1049 smp_mb();
1050 curr = pkc->nxt_offset;
1051 pkc->skb = skb;
e3192690 1052 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1053
1054 /* first try the current block */
1055 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1056 prb_fill_curr_block(curr, pkc, pbd, len);
1057 return (void *)curr;
1058 }
1059
1060 /* Ok, close the current block */
1061 prb_retire_current_block(pkc, po, 0);
1062
1063 /* Now, try to dispatch the next block */
1064 curr = (char *)prb_dispatch_next_block(pkc, po);
1065 if (curr) {
1066 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1067 prb_fill_curr_block(curr, pkc, pbd, len);
1068 return (void *)curr;
1069 }
1070
1071 /*
1072 * No free blocks are available.user_space hasn't caught up yet.
1073 * Queue was just frozen and now this packet will get dropped.
1074 */
1075 return NULL;
1076}
1077
eea49cc9 1078static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1079 struct sk_buff *skb,
1080 int status, unsigned int len)
1081{
1082 char *curr = NULL;
1083 switch (po->tp_version) {
1084 case TPACKET_V1:
1085 case TPACKET_V2:
1086 curr = packet_lookup_frame(po, &po->rx_ring,
1087 po->rx_ring.head, status);
1088 return curr;
1089 case TPACKET_V3:
1090 return __packet_lookup_frame_in_block(po, skb, status, len);
1091 default:
1092 WARN(1, "TPACKET version not supported\n");
1093 BUG();
99aa3473 1094 return NULL;
f6fb8f10 1095 }
1096}
1097
eea49cc9 1098static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1099 struct packet_ring_buffer *rb,
77f65ebd 1100 unsigned int idx,
f6fb8f10 1101 int status)
1102{
bc59ba39 1103 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1104 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1105
1106 if (status != BLOCK_STATUS(pbd))
1107 return NULL;
1108 return pbd;
1109}
1110
eea49cc9 1111static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1112{
1113 unsigned int prev;
1114 if (rb->prb_bdqc.kactive_blk_num)
1115 prev = rb->prb_bdqc.kactive_blk_num-1;
1116 else
1117 prev = rb->prb_bdqc.knum_blocks-1;
1118 return prev;
1119}
1120
1121/* Assumes caller has held the rx_queue.lock */
eea49cc9 1122static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1123 struct packet_ring_buffer *rb,
1124 int status)
1125{
1126 unsigned int previous = prb_previous_blk_num(rb);
1127 return prb_lookup_block(po, rb, previous, status);
1128}
1129
eea49cc9 1130static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1131 struct packet_ring_buffer *rb,
1132 int status)
1133{
1134 if (po->tp_version <= TPACKET_V2)
1135 return packet_previous_frame(po, rb, status);
1136
1137 return __prb_previous_block(po, rb, status);
1138}
1139
eea49cc9 1140static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1141 struct packet_ring_buffer *rb)
1142{
1143 switch (po->tp_version) {
1144 case TPACKET_V1:
1145 case TPACKET_V2:
1146 return packet_increment_head(rb);
1147 case TPACKET_V3:
1148 default:
1149 WARN(1, "TPACKET version not supported.\n");
1150 BUG();
1151 return;
1152 }
1153}
1154
eea49cc9 1155static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1156 struct packet_ring_buffer *rb,
1157 int status)
1158{
1159 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1160 return packet_lookup_frame(po, rb, previous, status);
1161}
1162
eea49cc9 1163static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1164{
1165 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1166}
1167
77f65ebd
WB		 1168static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1169{
1170 struct sock *sk = &po->sk;
1171 bool has_room;
1172
1173 if (po->prot_hook.func != tpacket_rcv)
1174 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1175 <= sk->sk_rcvbuf;
1176
1177 spin_lock(&sk->sk_receive_queue.lock);
1178 if (po->tp_version == TPACKET_V3)
1179 has_room = prb_lookup_block(po, &po->rx_ring,
1180 po->rx_ring.prb_bdqc.kactive_blk_num,
1181 TP_STATUS_KERNEL);
1182 else
1183 has_room = packet_lookup_frame(po, &po->rx_ring,
1184 po->rx_ring.head,
1185 TP_STATUS_KERNEL);
1186 spin_unlock(&sk->sk_receive_queue.lock);
1187
1188 return has_room;
1189}
1190
1da177e4
LT		 1191static void packet_sock_destruct(struct sock *sk)
1192{
ed85b565
RC		 1193 skb_queue_purge(&sk->sk_error_queue);
1194
547b792c
IJ		 1195 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1196 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1197
1198 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1199 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1200 return;
1201 }
1202
17ab56a2 1203 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1204}
1205
dc99f600
DM		 1206static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1207{
1208 int x = atomic_read(&f->rr_cur) + 1;
1209
1210 if (x >= num)
1211 x = 0;
1212
1213 return x;
1214}
1215
77f65ebd
WB		 1216static unsigned int fanout_demux_hash(struct packet_fanout *f,
1217 struct sk_buff *skb,
1218 unsigned int num)
dc99f600 1219{
f55d112e 1220 return reciprocal_divide(skb->rxhash, num);
dc99f600
DM		 1221}
1222
77f65ebd
WB		 1223static unsigned int fanout_demux_lb(struct packet_fanout *f,
1224 struct sk_buff *skb,
1225 unsigned int num)
dc99f600
DM		 1226{
1227 int cur, old;
1228
1229 cur = atomic_read(&f->rr_cur);
1230 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1231 fanout_rr_next(f, num))) != cur)
1232 cur = old;
77f65ebd
WB		 1233 return cur;
1234}
1235
1236static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1237 struct sk_buff *skb,
1238 unsigned int num)
1239{
1240 return smp_processor_id() % num;
dc99f600
DM		 1241}
1242
5df0ddfb
DB		 1243static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1244 struct sk_buff *skb,
1245 unsigned int num)
1246{
1247 return reciprocal_divide(prandom_u32(), num);
1248}
1249
77f65ebd
WB		 1250static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1251 struct sk_buff *skb,
1252 unsigned int idx, unsigned int skip,
1253 unsigned int num)
95ec3eb4 1254{
77f65ebd 1255 unsigned int i, j;
95ec3eb4 1256
77f65ebd
WB		 1257 i = j = min_t(int, f->next[idx], num - 1);
1258 do {
1259 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1260 if (i != j)
1261 f->next[idx] = i;
1262 return i;
1263 }
1264 if (++i == num)
1265 i = 0;
1266 } while (i != j);
1267
1268 return idx;
1269}
1270
1271static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1272{
1273 return f->flags & (flag >> 8);
95ec3eb4
DM		 1274}
1275
95ec3eb4
DM		 1276static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1277 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1278{
1279 struct packet_fanout *f = pt->af_packet_priv;
1280 unsigned int num = f->num_members;
1281 struct packet_sock *po;
77f65ebd 1282 unsigned int idx;
dc99f600
DM		 1283
1284 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1285 !num) {
1286 kfree_skb(skb);
1287 return 0;
1288 }
1289
95ec3eb4
DM		 1290 switch (f->type) {
1291 case PACKET_FANOUT_HASH:
1292 default:
77f65ebd 1293 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1294 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1295 if (!skb)
1296 return 0;
1297 }
3958afa1 1298 skb_get_hash(skb);
77f65ebd 1299 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1300 break;
1301 case PACKET_FANOUT_LB:
77f65ebd 1302 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1303 break;
1304 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1305 idx = fanout_demux_cpu(f, skb, num);
1306 break;
5df0ddfb
DB		 1307 case PACKET_FANOUT_RND:
1308 idx = fanout_demux_rnd(f, skb, num);
1309 break;
77f65ebd
WB		 1310 case PACKET_FANOUT_ROLLOVER:
1311 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1312 break;
dc99f600
DM		 1313 }
1314
77f65ebd
WB		 1315 po = pkt_sk(f->arr[idx]);
1316 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1317 unlikely(!packet_rcv_has_room(po, skb))) {
1318 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1319 po = pkt_sk(f->arr[idx]);
1320 }
dc99f600
DM		 1321
1322 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1323}
1324
fff3321d
PE		 1325DEFINE_MUTEX(fanout_mutex);
1326EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1327static LIST_HEAD(fanout_list);
1328
1329static void __fanout_link(struct sock *sk, struct packet_sock *po)
1330{
1331 struct packet_fanout *f = po->fanout;
1332
1333 spin_lock(&f->lock);
1334 f->arr[f->num_members] = sk;
1335 smp_wmb();
1336 f->num_members++;
1337 spin_unlock(&f->lock);
1338}
1339
1340static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1341{
1342 struct packet_fanout *f = po->fanout;
1343 int i;
1344
1345 spin_lock(&f->lock);
1346 for (i = 0; i < f->num_members; i++) {
1347 if (f->arr[i] == sk)
1348 break;
1349 }
1350 BUG_ON(i >= f->num_members);
1351 f->arr[i] = f->arr[f->num_members - 1];
1352 f->num_members--;
1353 spin_unlock(&f->lock);
1354}
1355
a0dfb263 1356static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
c0de08d0
EL		 1357{
1358 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1359 return true;
1360
1361 return false;
1362}
1363
7736d33f 1364static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1365{
1366 struct packet_sock *po = pkt_sk(sk);
1367 struct packet_fanout *f, *match;
7736d33f 1368 u8 type = type_flags & 0xff;
77f65ebd 1369 u8 flags = type_flags >> 8;
dc99f600
DM		 1370 int err;
1371
1372 switch (type) {
77f65ebd
WB		 1373 case PACKET_FANOUT_ROLLOVER:
1374 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1375 return -EINVAL;
dc99f600
DM		 1376 case PACKET_FANOUT_HASH:
1377 case PACKET_FANOUT_LB:
95ec3eb4 1378 case PACKET_FANOUT_CPU:
5df0ddfb 1379 case PACKET_FANOUT_RND:
dc99f600
DM		 1380 break;
1381 default:
1382 return -EINVAL;
1383 }
1384
1385 if (!po->running)
1386 return -EINVAL;
1387
1388 if (po->fanout)
1389 return -EALREADY;
1390
1391 mutex_lock(&fanout_mutex);
1392 match = NULL;
1393 list_for_each_entry(f, &fanout_list, list) {
1394 if (f->id == id &&
1395 read_pnet(&f->net) == sock_net(sk)) {
1396 match = f;
1397 break;
1398 }
1399 }
afe62c68 1400 err = -EINVAL;
77f65ebd 1401 if (match && match->flags != flags)
afe62c68 1402 goto out;
dc99f600 1403 if (!match) {
afe62c68 1404 err = -ENOMEM;
dc99f600 1405 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1406 if (!match)
1407 goto out;
1408 write_pnet(&match->net, sock_net(sk));
1409 match->id = id;
1410 match->type = type;
77f65ebd 1411 match->flags = flags;
afe62c68
ED		 1412 atomic_set(&match->rr_cur, 0);
1413 INIT_LIST_HEAD(&match->list);
1414 spin_lock_init(&match->lock);
1415 atomic_set(&match->sk_ref, 0);
1416 match->prot_hook.type = po->prot_hook.type;
1417 match->prot_hook.dev = po->prot_hook.dev;
1418 match->prot_hook.func = packet_rcv_fanout;
1419 match->prot_hook.af_packet_priv = match;
c0de08d0 1420 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1421 dev_add_pack(&match->prot_hook);
1422 list_add(&match->list, &fanout_list);
dc99f600 1423 }
afe62c68
ED		 1424 err = -EINVAL;
1425 if (match->type == type &&
1426 match->prot_hook.type == po->prot_hook.type &&
1427 match->prot_hook.dev == po->prot_hook.dev) {
1428 err = -ENOSPC;
1429 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1430 __dev_remove_pack(&po->prot_hook);
1431 po->fanout = match;
1432 atomic_inc(&match->sk_ref);
1433 __fanout_link(sk, po);
1434 err = 0;
dc99f600
DM		 1435 }
1436 }
afe62c68 1437out:
dc99f600
DM		 1438 mutex_unlock(&fanout_mutex);
1439 return err;
1440}
1441
1442static void fanout_release(struct sock *sk)
1443{
1444 struct packet_sock *po = pkt_sk(sk);
1445 struct packet_fanout *f;
1446
1447 f = po->fanout;
1448 if (!f)
1449 return;
1450
fff3321d 1451 mutex_lock(&fanout_mutex);
dc99f600
DM		 1452 po->fanout = NULL;
1453
dc99f600
DM		 1454 if (atomic_dec_and_test(&f->sk_ref)) {
1455 list_del(&f->list);
1456 dev_remove_pack(&f->prot_hook);
1457 kfree(f);
1458 }
1459 mutex_unlock(&fanout_mutex);
1460}
1da177e4 1461
90ddc4f0 1462static const struct proto_ops packet_ops;
1da177e4 1463
90ddc4f0 1464static const struct proto_ops packet_ops_spkt;
1da177e4 1465
40d4e3df
ED		 1466static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1467 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1468{
1469 struct sock *sk;
1470 struct sockaddr_pkt *spkt;
1471
1472 /*
1473 * When we registered the protocol we saved the socket in the data
1474 * field for just this event.
1475 */
1476
1477 sk = pt->af_packet_priv;
1ce4f28b 1478
1da177e4
LT		 1479 /*
1480 * Yank back the headers [hope the device set this
1481 * right or kerboom...]
1482 *
1483 * Incoming packets have ll header pulled,
1484 * push it back.
1485 *
98e399f8 1486 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1487 * so that this procedure is noop.
1488 */
1489
1490 if (skb->pkt_type == PACKET_LOOPBACK)
1491 goto out;
1492
09ad9bc7 1493 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1494 goto out;
1495
40d4e3df
ED		 1496 skb = skb_share_check(skb, GFP_ATOMIC);
1497 if (skb == NULL)
1da177e4
LT		 1498 goto oom;
1499
1500 /* drop any routing info */
adf30907 1501 skb_dst_drop(skb);
1da177e4 1502
84531c24
PO		 1503 /* drop conntrack reference */
1504 nf_reset(skb);
1505
ffbc6111 1506 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1507
98e399f8 1508 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1509
1510 /*
1511 * The SOCK_PACKET socket receives _all_ frames.
1512 */
1513
1514 spkt->spkt_family = dev->type;
1515 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1516 spkt->spkt_protocol = skb->protocol;
1517
1518 /*
1519 * Charge the memory to the socket. This is done specifically
1520 * to prevent sockets using all the memory up.
1521 */
1522
40d4e3df 1523 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1524 return 0;
1525
1526out:
1527 kfree_skb(skb);
1528oom:
1529 return 0;
1530}
1531
1532
1533/*
1534 * Output a raw packet to a device layer. This bypasses all the other
1535 * protocol layers and you must therefore supply it with a complete frame
1536 */
1ce4f28b 1537
1da177e4
LT		 1538static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1539 struct msghdr *msg, size_t len)
1540{
1541 struct sock *sk = sock->sk;
40d4e3df 1542 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1543 struct sk_buff *skb = NULL;
1da177e4 1544 struct net_device *dev;
40d4e3df 1545 __be16 proto = 0;
1da177e4 1546 int err;
3bdc0eba 1547 int extra_len = 0;
1ce4f28b 1548
1da177e4 1549 /*
1ce4f28b 1550 * Get and verify the address.
1da177e4
LT		 1551 */
1552
40d4e3df 1553 if (saddr) {
1da177e4 1554 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1555 return -EINVAL;
1556 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1557 proto = saddr->spkt_protocol;
1558 } else
1559 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1560
1561 /*
1ce4f28b 1562 * Find the device first to size check it
1da177e4
LT		 1563 */
1564
de74e92a 1565 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1566retry:
654d1f8a
ED		 1567 rcu_read_lock();
1568 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1569 err = -ENODEV;
1570 if (dev == NULL)
1571 goto out_unlock;
1ce4f28b 1572
d5e76b0a
DM		 1573 err = -ENETDOWN;
1574 if (!(dev->flags & IFF_UP))
1575 goto out_unlock;
1576
1da177e4 1577 /*
40d4e3df
ED		 1578 * You may not queue a frame bigger than the mtu. This is the lowest level
1579 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1580 */
1ce4f28b 1581
3bdc0eba
BG		 1582 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1583 if (!netif_supports_nofcs(dev)) {
1584 err = -EPROTONOSUPPORT;
1585 goto out_unlock;
1586 }
1587 extra_len = 4; /* We're doing our own CRC */
1588 }
1589
1da177e4 1590 err = -EMSGSIZE;
3bdc0eba 1591 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1592 goto out_unlock;
1593
1a35ca80
ED		 1594 if (!skb) {
1595 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1596 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1597 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1598
1599 rcu_read_unlock();
4ce40912 1600 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1601 if (skb == NULL)
1602 return -ENOBUFS;
1603 /* FIXME: Save some space for broken drivers that write a hard
1604 * header at transmission time by themselves. PPP is the notable
1605 * one here. This should really be fixed at the driver level.
1606 */
1607 skb_reserve(skb, reserved);
1608 skb_reset_network_header(skb);
1609
1610 /* Try to align data part correctly */
1611 if (hhlen) {
1612 skb->data -= hhlen;
1613 skb->tail -= hhlen;
1614 if (len < hhlen)
1615 skb_reset_network_header(skb);
1616 }
1617 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1618 if (err)
1619 goto out_free;
1620 goto retry;
1da177e4
LT		 1621 }
1622
3bdc0eba 1623 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1624 /* Earlier code assumed this would be a VLAN pkt,
1625 * double-check this now that we have the actual
1626 * packet in hand.
1627 */
1628 struct ethhdr *ehdr;
1629 skb_reset_mac_header(skb);
1630 ehdr = eth_hdr(skb);
1631 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1632 err = -EMSGSIZE;
1633 goto out_unlock;
1634 }
1635 }
1a35ca80 1636
1da177e4
LT		 1637 skb->protocol = proto;
1638 skb->dev = dev;
1639 skb->priority = sk->sk_priority;
2d37a186 1640 skb->mark = sk->sk_mark;
bf84a010
DB		 1641
1642 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1643
3bdc0eba
BG		 1644 if (unlikely(extra_len == 4))
1645 skb->no_fcs = 1;
1646
40893fd0 1647 skb_probe_transport_header(skb, 0);
c1aad275 1648
1da177e4 1649 dev_queue_xmit(skb);
654d1f8a 1650 rcu_read_unlock();
40d4e3df 1651 return len;
1da177e4 1652
1da177e4 1653out_unlock:
654d1f8a 1654 rcu_read_unlock();
1a35ca80
ED		 1655out_free:
1656 kfree_skb(skb);
1da177e4
LT		 1657 return err;
1658}
1da177e4 1659
eea49cc9 1660static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1661 const struct sock *sk,
dbcb5855 1662 unsigned int res)
1da177e4
LT		 1663{
1664 struct sk_filter *filter;
fda9ef5d 1665
80f8f102
ED		 1666 rcu_read_lock();
1667 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1668 if (filter != NULL)
0a14842f 1669 res = SK_RUN_FILTER(filter, skb);
80f8f102 1670 rcu_read_unlock();
1da177e4 1671
dbcb5855 1672 return res;
1da177e4
LT		 1673}
1674
1675/*
62ab0812
ED		 1676 * This function makes lazy skb cloning in hope that most of packets
1677 * are discarded by BPF.
1678 *
1679 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1680 * and skb->cb are mangled. It works because (and until) packets
1681 * falling here are owned by current CPU. Output packets are cloned
1682 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1683 * sequencially, so that if we return skb to original state on exit,
1684 * we will not harm anyone.
1da177e4
LT		 1685 */
1686
40d4e3df
ED		 1687static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1688 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1689{
1690 struct sock *sk;
1691 struct sockaddr_ll *sll;
1692 struct packet_sock *po;
40d4e3df 1693 u8 *skb_head = skb->data;
1da177e4 1694 int skb_len = skb->len;
dbcb5855 1695 unsigned int snaplen, res;
1da177e4
LT		 1696
1697 if (skb->pkt_type == PACKET_LOOPBACK)
1698 goto drop;
1699
1700 sk = pt->af_packet_priv;
1701 po = pkt_sk(sk);
1702
09ad9bc7 1703 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1704 goto drop;
1705
1da177e4
LT		 1706 skb->dev = dev;
1707
3b04ddde 1708 if (dev->header_ops) {
1da177e4 1709 /* The device has an explicit notion of ll header,
62ab0812
ED		 1710 * exported to higher levels.
1711 *
1712 * Otherwise, the device hides details of its frame
1713 * structure, so that corresponding packet head is
1714 * never delivered to user.
1da177e4
LT		 1715 */
1716 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1717 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1718 else if (skb->pkt_type == PACKET_OUTGOING) {
1719 /* Special case: outgoing packets have ll header at head */
bbe735e4 1720 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1721 }
1722 }
1723
1724 snaplen = skb->len;
1725
dbcb5855
DM		 1726 res = run_filter(skb, sk, snaplen);
1727 if (!res)
fda9ef5d 1728 goto drop_n_restore;
dbcb5855
DM		 1729 if (snaplen > res)
1730 snaplen = res;
1da177e4 1731
0fd7bac6 1732 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1733 goto drop_n_acct;
1734
1735 if (skb_shared(skb)) {
1736 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1737 if (nskb == NULL)
1738 goto drop_n_acct;
1739
1740 if (skb_head != skb->data) {
1741 skb->data = skb_head;
1742 skb->len = skb_len;
1743 }
abc4e4fa 1744 consume_skb(skb);
1da177e4
LT		 1745 skb = nskb;
1746 }
1747
ffbc6111
HX		 1748 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1749 sizeof(skb->cb));
1750
1751 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1752 sll->sll_family = AF_PACKET;
1753 sll->sll_hatype = dev->type;
1754 sll->sll_protocol = skb->protocol;
1755 sll->sll_pkttype = skb->pkt_type;
8032b464 1756 if (unlikely(po->origdev))
80feaacb
PWJ		 1757 sll->sll_ifindex = orig_dev->ifindex;
1758 else
1759 sll->sll_ifindex = dev->ifindex;
1da177e4 1760
b95cce35 1761 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1762
ffbc6111 1763 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1764
1da177e4
LT		 1765 if (pskb_trim(skb, snaplen))
1766 goto drop_n_acct;
1767
1768 skb_set_owner_r(skb, sk);
1769 skb->dev = NULL;
adf30907 1770 skb_dst_drop(skb);
1da177e4 1771
84531c24
PO		 1772 /* drop conntrack reference */
1773 nf_reset(skb);
1774
1da177e4 1775 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1776 po->stats.stats1.tp_packets++;
3b885787 1777 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1778 __skb_queue_tail(&sk->sk_receive_queue, skb);
1779 spin_unlock(&sk->sk_receive_queue.lock);
1780 sk->sk_data_ready(sk, skb->len);
1781 return 0;
1782
1783drop_n_acct:
7091fbd8 1784 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1785 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1786 atomic_inc(&sk->sk_drops);
1787 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1788
1789drop_n_restore:
1790 if (skb_head != skb->data && skb_shared(skb)) {
1791 skb->data = skb_head;
1792 skb->len = skb_len;
1793 }
1794drop:
ead2ceb0 1795 consume_skb(skb);
1da177e4
LT		 1796 return 0;
1797}
1798
40d4e3df
ED		 1799static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1800 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1801{
1802 struct sock *sk;
1803 struct packet_sock *po;
1804 struct sockaddr_ll *sll;
184f489e 1805 union tpacket_uhdr h;
40d4e3df 1806 u8 *skb_head = skb->data;
1da177e4 1807 int skb_len = skb->len;
dbcb5855 1808 unsigned int snaplen, res;
f6fb8f10 1809 unsigned long status = TP_STATUS_USER;
bbd6ef87 1810 unsigned short macoff, netoff, hdrlen;
1da177e4 1811 struct sk_buff *copy_skb = NULL;
bbd6ef87 1812 struct timespec ts;
b9c32fb2 1813 __u32 ts_status;
1da177e4 1814
51846355
AW		 1815 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1816 * We may add members to them until current aligned size without forcing
1817 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1818 */
1819 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1820 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1821
1da177e4
LT		 1822 if (skb->pkt_type == PACKET_LOOPBACK)
1823 goto drop;
1824
1825 sk = pt->af_packet_priv;
1826 po = pkt_sk(sk);
1827
09ad9bc7 1828 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1829 goto drop;
1830
3b04ddde 1831 if (dev->header_ops) {
1da177e4 1832 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1833 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1834 else if (skb->pkt_type == PACKET_OUTGOING) {
1835 /* Special case: outgoing packets have ll header at head */
bbe735e4 1836 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1837 }
1838 }
1839
8dc41944
HX		 1840 if (skb->ip_summed == CHECKSUM_PARTIAL)
1841 status |= TP_STATUS_CSUMNOTREADY;
1842
1da177e4
LT		 1843 snaplen = skb->len;
1844
dbcb5855
DM		 1845 res = run_filter(skb, sk, snaplen);
1846 if (!res)
fda9ef5d 1847 goto drop_n_restore;
dbcb5855
DM		 1848 if (snaplen > res)
1849 snaplen = res;
1da177e4
LT		 1850
1851 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1852 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1853 po->tp_reserve;
1da177e4 1854 } else {
95c96174 1855 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1856 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1857 (maclen < 16 ? 16 : maclen)) +
1858 po->tp_reserve;
1da177e4
LT		 1859 macoff = netoff - maclen;
1860 }
f6fb8f10 1861 if (po->tp_version <= TPACKET_V2) {
1862 if (macoff + snaplen > po->rx_ring.frame_size) {
1863 if (po->copy_thresh &&
0fd7bac6 1864 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1865 if (skb_shared(skb)) {
1866 copy_skb = skb_clone(skb, GFP_ATOMIC);
1867 } else {
1868 copy_skb = skb_get(skb);
1869 skb_head = skb->data;
1870 }
1871 if (copy_skb)
1872 skb_set_owner_r(copy_skb, sk);
1da177e4 1873 }
f6fb8f10 1874 snaplen = po->rx_ring.frame_size - macoff;
1875 if ((int)snaplen < 0)
1876 snaplen = 0;
1da177e4 1877 }
1da177e4 1878 }
1da177e4 1879 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1880 h.raw = packet_current_rx_frame(po, skb,
1881 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1882 if (!h.raw)
1da177e4 1883 goto ring_is_full;
f6fb8f10 1884 if (po->tp_version <= TPACKET_V2) {
1885 packet_increment_rx_head(po, &po->rx_ring);
1886 /*
1887 * LOSING will be reported till you read the stats,
1888 * because it's COR - Clear On Read.
1889 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1890 * at packet level.
1891 */
ee80fbf3 1892 if (po->stats.stats1.tp_drops)
f6fb8f10 1893 status |= TP_STATUS_LOSING;
1894 }
ee80fbf3 1895 po->stats.stats1.tp_packets++;
1da177e4
LT		 1896 if (copy_skb) {
1897 status |= TP_STATUS_COPY;
1898 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1899 }
1da177e4
LT		 1900 spin_unlock(&sk->sk_receive_queue.lock);
1901
bbd6ef87 1902 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1903
1904 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1905 getnstimeofday(&ts);
1da177e4 1906
b9c32fb2
DB		 1907 status |= ts_status;
1908
bbd6ef87
PM		 1909 switch (po->tp_version) {
1910 case TPACKET_V1:
1911 h.h1->tp_len = skb->len;
1912 h.h1->tp_snaplen = snaplen;
1913 h.h1->tp_mac = macoff;
1914 h.h1->tp_net = netoff;
4b457bdf
DB		 1915 h.h1->tp_sec = ts.tv_sec;
1916 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1917 hdrlen = sizeof(*h.h1);
1918 break;
1919 case TPACKET_V2:
1920 h.h2->tp_len = skb->len;
1921 h.h2->tp_snaplen = snaplen;
1922 h.h2->tp_mac = macoff;
1923 h.h2->tp_net = netoff;
bbd6ef87
PM		 1924 h.h2->tp_sec = ts.tv_sec;
1925 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1926 if (vlan_tx_tag_present(skb)) {
1927 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1928 status |= TP_STATUS_VLAN_VALID;
1929 } else {
1930 h.h2->tp_vlan_tci = 0;
1931 }
e4d26f4b 1932 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 1933 hdrlen = sizeof(*h.h2);
1934 break;
f6fb8f10 1935 case TPACKET_V3:
1936 /* tp_nxt_offset,vlan are already populated above.
1937 * So DONT clear those fields here
1938 */
1939 h.h3->tp_status |= status;
1940 h.h3->tp_len = skb->len;
1941 h.h3->tp_snaplen = snaplen;
1942 h.h3->tp_mac = macoff;
1943 h.h3->tp_net = netoff;
f6fb8f10 1944 h.h3->tp_sec = ts.tv_sec;
1945 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 1946 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 1947 hdrlen = sizeof(*h.h3);
1948 break;
bbd6ef87
PM		 1949 default:
1950 BUG();
1951 }
1da177e4 1952
bbd6ef87 1953 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1954 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1955 sll->sll_family = AF_PACKET;
1956 sll->sll_hatype = dev->type;
1957 sll->sll_protocol = skb->protocol;
1958 sll->sll_pkttype = skb->pkt_type;
8032b464 1959 if (unlikely(po->origdev))
80feaacb
PWJ		 1960 sll->sll_ifindex = orig_dev->ifindex;
1961 else
1962 sll->sll_ifindex = dev->ifindex;
1da177e4 1963
e16aa207 1964 smp_mb();
f6dafa95 1965#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1966 {
0af55bb5
CG		 1967 u8 *start, *end;
1968
f6fb8f10 1969 if (po->tp_version <= TPACKET_V2) {
1970 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1971 + macoff + snaplen);
1972 for (start = h.raw; start < end; start += PAGE_SIZE)
1973 flush_dcache_page(pgv_to_page(start));
1974 }
cc9f01b2 1975 smp_wmb();
1da177e4 1976 }
f6dafa95 1977#endif
f6fb8f10 1978 if (po->tp_version <= TPACKET_V2)
1979 __packet_set_status(po, h.raw, status);
1980 else
1981 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1982
1983 sk->sk_data_ready(sk, 0);
1984
1985drop_n_restore:
1986 if (skb_head != skb->data && skb_shared(skb)) {
1987 skb->data = skb_head;
1988 skb->len = skb_len;
1989 }
1990drop:
1ce4f28b 1991 kfree_skb(skb);
1da177e4
LT		 1992 return 0;
1993
1994ring_is_full:
ee80fbf3 1995 po->stats.stats1.tp_drops++;
1da177e4
LT		 1996 spin_unlock(&sk->sk_receive_queue.lock);
1997
1998 sk->sk_data_ready(sk, 0);
acb5d75b 1999 kfree_skb(copy_skb);
1da177e4
LT		 2000 goto drop_n_restore;
2001}
2002
69e3c75f
JB		 2003static void tpacket_destruct_skb(struct sk_buff *skb)
2004{
2005 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 2006 void *ph;
1da177e4 2007
69e3c75f 2008 if (likely(po->tx_ring.pg_vec)) {
b9c32fb2
DB		 2009 __u32 ts;
2010
69e3c75f 2011 ph = skb_shinfo(skb)->destructor_arg;
69e3c75f
JB		 2012 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
2013 atomic_dec(&po->tx_ring.pending);
b9c32fb2
DB		 2014
2015 ts = __packet_set_timestamp(po, ph, skb);
2016 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2017 }
2018
2019 sock_wfree(skb);
2020}
2021
40d4e3df
ED		 2022static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2023 void *frame, struct net_device *dev, int size_max,
ae641949 2024 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 2025{
184f489e 2026 union tpacket_uhdr ph;
09effa67 2027 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 2028 struct socket *sock = po->sk.sk_socket;
2029 struct page *page;
2030 void *data;
2031 int err;
2032
2033 ph.raw = frame;
2034
2035 skb->protocol = proto;
2036 skb->dev = dev;
2037 skb->priority = po->sk.sk_priority;
2d37a186 2038 skb->mark = po->sk.sk_mark;
2e31396f 2039 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2040 skb_shinfo(skb)->destructor_arg = ph.raw;
2041
2042 switch (po->tp_version) {
2043 case TPACKET_V2:
2044 tp_len = ph.h2->tp_len;
2045 break;
2046 default:
2047 tp_len = ph.h1->tp_len;
2048 break;
2049 }
09effa67
DM		 2050 if (unlikely(tp_len > size_max)) {
2051 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2052 return -EMSGSIZE;
2053 }
69e3c75f 2054
ae641949 2055 skb_reserve(skb, hlen);
69e3c75f 2056 skb_reset_network_header(skb);
c1aad275 2057
d346a3fa
DB		 2058 if (!packet_use_direct_xmit(po))
2059 skb_probe_transport_header(skb, 0);
2060 if (unlikely(po->tp_tx_has_off)) {
5920cd3a
PC		 2061 int off_min, off_max, off;
2062 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2063 off_max = po->tx_ring.frame_size - tp_len;
2064 if (sock->type == SOCK_DGRAM) {
2065 switch (po->tp_version) {
2066 case TPACKET_V2:
2067 off = ph.h2->tp_net;
2068 break;
2069 default:
2070 off = ph.h1->tp_net;
2071 break;
2072 }
2073 } else {
2074 switch (po->tp_version) {
2075 case TPACKET_V2:
2076 off = ph.h2->tp_mac;
2077 break;
2078 default:
2079 off = ph.h1->tp_mac;
2080 break;
2081 }
2082 }
2083 if (unlikely((off < off_min) || (off_max < off)))
2084 return -EINVAL;
2085 data = ph.raw + off;
2086 } else {
2087 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2088 }
69e3c75f
JB		 2089 to_write = tp_len;
2090
2091 if (sock->type == SOCK_DGRAM) {
2092 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2093 NULL, tp_len);
2094 if (unlikely(err < 0))
2095 return -EINVAL;
40d4e3df 2096 } else if (dev->hard_header_len) {
69e3c75f
JB		 2097 /* net device doesn't like empty head */
2098 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2099 pr_err("packet size is too short (%d < %d)\n",
2100 tp_len, dev->hard_header_len);
69e3c75f
JB		 2101 return -EINVAL;
2102 }
2103
2104 skb_push(skb, dev->hard_header_len);
2105 err = skb_store_bits(skb, 0, data,
2106 dev->hard_header_len);
2107 if (unlikely(err))
2108 return err;
2109
2110 data += dev->hard_header_len;
2111 to_write -= dev->hard_header_len;
2112 }
2113
69e3c75f
JB		 2114 offset = offset_in_page(data);
2115 len_max = PAGE_SIZE - offset;
2116 len = ((to_write > len_max) ? len_max : to_write);
2117
2118 skb->data_len = to_write;
2119 skb->len += to_write;
2120 skb->truesize += to_write;
2121 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2122
2123 while (likely(to_write)) {
2124 nr_frags = skb_shinfo(skb)->nr_frags;
2125
2126 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2127 pr_err("Packet exceed the number of skb frags(%lu)\n",
2128 MAX_SKB_FRAGS);
69e3c75f
JB		 2129 return -EFAULT;
2130 }
2131
0af55bb5
CG		 2132 page = pgv_to_page(data);
2133 data += len;
69e3c75f
JB		 2134 flush_dcache_page(page);
2135 get_page(page);
0af55bb5 2136 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2137 to_write -= len;
2138 offset = 0;
2139 len_max = PAGE_SIZE;
2140 len = ((to_write > len_max) ? len_max : to_write);
2141 }
2142
2143 return tp_len;
2144}
2145
2146static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2147{
69e3c75f
JB		 2148 struct sk_buff *skb;
2149 struct net_device *dev;
2150 __be16 proto;
09effa67 2151 int err, reserve = 0;
40d4e3df
ED		 2152 void *ph;
2153 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2154 int tp_len, size_max;
2155 unsigned char *addr;
2156 int len_sum = 0;
9e67030a 2157 int status = TP_STATUS_AVAILABLE;
ae641949 2158 int hlen, tlen;
69e3c75f 2159
69e3c75f
JB		 2160 mutex_lock(&po->pg_vec_lock);
2161
66e56cd4 2162 if (likely(saddr == NULL)) {
e40526cb 2163 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2164 proto = po->num;
2165 addr = NULL;
2166 } else {
2167 err = -EINVAL;
2168 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2169 goto out;
2170 if (msg->msg_namelen < (saddr->sll_halen
2171 + offsetof(struct sockaddr_ll,
2172 sll_addr)))
2173 goto out;
69e3c75f
JB		 2174 proto = saddr->sll_protocol;
2175 addr = saddr->sll_addr;
827d9780 2176 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2177 }
2178
69e3c75f
JB		 2179 err = -ENXIO;
2180 if (unlikely(dev == NULL))
2181 goto out;
69e3c75f
JB		 2182 err = -ENETDOWN;
2183 if (unlikely(!(dev->flags & IFF_UP)))
2184 goto out_put;
2185
e40526cb
DB		 2186 reserve = dev->hard_header_len;
2187
69e3c75f 2188 size_max = po->tx_ring.frame_size
b5dd884e 2189 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2190
09effa67
DM		 2191 if (size_max > dev->mtu + reserve)
2192 size_max = dev->mtu + reserve;
2193
69e3c75f
JB		 2194 do {
2195 ph = packet_current_frame(po, &po->tx_ring,
2196 TP_STATUS_SEND_REQUEST);
2197
2198 if (unlikely(ph == NULL)) {
2199 schedule();
2200 continue;
2201 }
2202
2203 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2204 hlen = LL_RESERVED_SPACE(dev);
2205 tlen = dev->needed_tailroom;
69e3c75f 2206 skb = sock_alloc_send_skb(&po->sk,
ae641949 2207 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2208 0, &err);
2209
2210 if (unlikely(skb == NULL))
2211 goto out_status;
2212
2213 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2214 addr, hlen);
69e3c75f
JB		 2215
2216 if (unlikely(tp_len < 0)) {
2217 if (po->tp_loss) {
2218 __packet_set_status(po, ph,
2219 TP_STATUS_AVAILABLE);
2220 packet_increment_head(&po->tx_ring);
2221 kfree_skb(skb);
2222 continue;
2223 } else {
2224 status = TP_STATUS_WRONG_FORMAT;
2225 err = tp_len;
2226 goto out_status;
2227 }
2228 }
2229
d346a3fa 2230 skb_set_queue_mapping(skb, packet_pick_tx_queue(dev));
69e3c75f
JB		 2231 skb->destructor = tpacket_destruct_skb;
2232 __packet_set_status(po, ph, TP_STATUS_SENDING);
2233 atomic_inc(&po->tx_ring.pending);
2234
2235 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2236 err = po->xmit(skb);
eb70df13
JP		 2237 if (unlikely(err > 0)) {
2238 err = net_xmit_errno(err);
2239 if (err && __packet_get_status(po, ph) ==
2240 TP_STATUS_AVAILABLE) {
2241 /* skb was destructed already */
2242 skb = NULL;
2243 goto out_status;
2244 }
2245 /*
2246 * skb was dropped but not destructed yet;
2247 * let's treat it like congestion or err < 0
2248 */
2249 err = 0;
2250 }
69e3c75f
JB		 2251 packet_increment_head(&po->tx_ring);
2252 len_sum += tp_len;
f64f9e71
JP		 2253 } while (likely((ph != NULL) ||
2254 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2255 (atomic_read(&po->tx_ring.pending))))
2256 );
69e3c75f
JB		 2257
2258 err = len_sum;
2259 goto out_put;
2260
69e3c75f
JB		 2261out_status:
2262 __packet_set_status(po, ph, status);
2263 kfree_skb(skb);
2264out_put:
e40526cb 2265 dev_put(dev);
69e3c75f
JB		 2266out:
2267 mutex_unlock(&po->pg_vec_lock);
2268 return err;
2269}
69e3c75f 2270
eea49cc9
OJ		 2271static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2272 size_t reserve, size_t len,
2273 size_t linear, int noblock,
2274 int *err)
bfd5f4a3
SS		 2275{
2276 struct sk_buff *skb;
2277
2278 /* Under a page? Don't bother with paged skb. */
2279 if (prepad + len < PAGE_SIZE || !linear)
2280 linear = len;
2281
2282 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2283 err, 0);
bfd5f4a3
SS		 2284 if (!skb)
2285 return NULL;
2286
2287 skb_reserve(skb, reserve);
2288 skb_put(skb, linear);
2289 skb->data_len = len - linear;
2290 skb->len += len - linear;
2291
2292 return skb;
2293}
2294
d346a3fa 2295static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2296{
2297 struct sock *sk = sock->sk;
40d4e3df 2298 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2299 struct sk_buff *skb;
2300 struct net_device *dev;
0e11c91e 2301 __be16 proto;
1da177e4 2302 unsigned char *addr;
827d9780 2303 int err, reserve = 0;
bfd5f4a3
SS		 2304 struct virtio_net_hdr vnet_hdr = { 0 };
2305 int offset = 0;
2306 int vnet_hdr_len;
2307 struct packet_sock *po = pkt_sk(sk);
2308 unsigned short gso_type = 0;
ae641949 2309 int hlen, tlen;
3bdc0eba 2310 int extra_len = 0;
1da177e4
LT		 2311
2312 /*
1ce4f28b 2313 * Get and verify the address.
1da177e4 2314 */
1ce4f28b 2315
66e56cd4 2316 if (likely(saddr == NULL)) {
e40526cb 2317 dev = packet_cached_dev_get(po);
1da177e4
LT		 2318 proto = po->num;
2319 addr = NULL;
2320 } else {
2321 err = -EINVAL;
2322 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2323 goto out;
0fb375fb
EB		 2324 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2325 goto out;
1da177e4
LT		 2326 proto = saddr->sll_protocol;
2327 addr = saddr->sll_addr;
827d9780 2328 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2329 }
2330
1da177e4 2331 err = -ENXIO;
e40526cb 2332 if (unlikely(dev == NULL))
1da177e4 2333 goto out_unlock;
d5e76b0a 2334 err = -ENETDOWN;
e40526cb 2335 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2336 goto out_unlock;
2337
e40526cb
DB		 2338 if (sock->type == SOCK_RAW)
2339 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2340 if (po->has_vnet_hdr) {
2341 vnet_hdr_len = sizeof(vnet_hdr);
2342
2343 err = -EINVAL;
2344 if (len < vnet_hdr_len)
2345 goto out_unlock;
2346
2347 len -= vnet_hdr_len;
2348
2349 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2350 vnet_hdr_len);
2351 if (err < 0)
2352 goto out_unlock;
2353
2354 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2355 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2356 vnet_hdr.hdr_len))
2357 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2358 vnet_hdr.csum_offset + 2;
2359
2360 err = -EINVAL;
2361 if (vnet_hdr.hdr_len > len)
2362 goto out_unlock;
2363
2364 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2365 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2366 case VIRTIO_NET_HDR_GSO_TCPV4:
2367 gso_type = SKB_GSO_TCPV4;
2368 break;
2369 case VIRTIO_NET_HDR_GSO_TCPV6:
2370 gso_type = SKB_GSO_TCPV6;
2371 break;
2372 case VIRTIO_NET_HDR_GSO_UDP:
2373 gso_type = SKB_GSO_UDP;
2374 break;
2375 default:
2376 goto out_unlock;
2377 }
2378
2379 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2380 gso_type |= SKB_GSO_TCP_ECN;
2381
2382 if (vnet_hdr.gso_size == 0)
2383 goto out_unlock;
2384
2385 }
2386 }
2387
3bdc0eba
BG		 2388 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2389 if (!netif_supports_nofcs(dev)) {
2390 err = -EPROTONOSUPPORT;
2391 goto out_unlock;
2392 }
2393 extra_len = 4; /* We're doing our own CRC */
2394 }
2395
1da177e4 2396 err = -EMSGSIZE;
3bdc0eba 2397 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2398 goto out_unlock;
2399
bfd5f4a3 2400 err = -ENOBUFS;
ae641949
HX		 2401 hlen = LL_RESERVED_SPACE(dev);
2402 tlen = dev->needed_tailroom;
2403 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2404 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2405 if (skb == NULL)
1da177e4
LT		 2406 goto out_unlock;
2407
bfd5f4a3 2408 skb_set_network_header(skb, reserve);
1da177e4 2409
0c4e8581
SH		 2410 err = -EINVAL;
2411 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2412 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2413 goto out_free;
1da177e4
LT		 2414
2415 /* Returns -EFAULT on error */
bfd5f4a3 2416 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2417 if (err)
2418 goto out_free;
bf84a010
DB		 2419
2420 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2421
3bdc0eba 2422 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2423 /* Earlier code assumed this would be a VLAN pkt,
2424 * double-check this now that we have the actual
2425 * packet in hand.
2426 */
2427 struct ethhdr *ehdr;
2428 skb_reset_mac_header(skb);
2429 ehdr = eth_hdr(skb);
2430 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2431 err = -EMSGSIZE;
2432 goto out_free;
2433 }
57f89bfa
BG		 2434 }
2435
09effa67
DM		 2436 skb->protocol = proto;
2437 skb->dev = dev;
1da177e4 2438 skb->priority = sk->sk_priority;
2d37a186 2439 skb->mark = sk->sk_mark;
d346a3fa 2440 skb_set_queue_mapping(skb, packet_pick_tx_queue(dev));
1da177e4 2441
bfd5f4a3
SS		 2442 if (po->has_vnet_hdr) {
2443 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2444 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2445 vnet_hdr.csum_offset)) {
2446 err = -EINVAL;
2447 goto out_free;
2448 }
2449 }
2450
2451 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2452 skb_shinfo(skb)->gso_type = gso_type;
2453
2454 /* Header must be checked, and gso_segs computed. */
2455 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2456 skb_shinfo(skb)->gso_segs = 0;
2457
2458 len += vnet_hdr_len;
2459 }
2460
d346a3fa
DB		 2461 if (!packet_use_direct_xmit(po))
2462 skb_probe_transport_header(skb, reserve);
3bdc0eba
BG		 2463 if (unlikely(extra_len == 4))
2464 skb->no_fcs = 1;
2465
d346a3fa 2466 err = po->xmit(skb);
1da177e4
LT		 2467 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2468 goto out_unlock;
2469
e40526cb 2470 dev_put(dev);
1da177e4 2471
40d4e3df 2472 return len;
1da177e4
LT		 2473
2474out_free:
2475 kfree_skb(skb);
2476out_unlock:
e40526cb 2477 if (dev)
1da177e4
LT		 2478 dev_put(dev);
2479out:
2480 return err;
2481}
2482
69e3c75f
JB		 2483static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2484 struct msghdr *msg, size_t len)
2485{
69e3c75f
JB		 2486 struct sock *sk = sock->sk;
2487 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2488
69e3c75f
JB		 2489 if (po->tx_ring.pg_vec)
2490 return tpacket_snd(po, msg);
2491 else
69e3c75f
JB		 2492 return packet_snd(sock, msg, len);
2493}
2494
1da177e4
LT		 2495/*
2496 * Close a PACKET socket. This is fairly simple. We immediately go
2497 * to 'closed' state and remove our protocol entry in the device list.
2498 */
2499
2500static int packet_release(struct socket *sock)
2501{
2502 struct sock *sk = sock->sk;
2503 struct packet_sock *po;
d12d01d6 2504 struct net *net;
f6fb8f10 2505 union tpacket_req_u req_u;
1da177e4
LT		 2506
2507 if (!sk)
2508 return 0;
2509
3b1e0a65 2510 net = sock_net(sk);
1da177e4
LT		 2511 po = pkt_sk(sk);
2512
0fa7fa98 2513 mutex_lock(&net->packet.sklist_lock);
808f5114 2514 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2515 mutex_unlock(&net->packet.sklist_lock);
2516
2517 preempt_disable();
920de804 2518 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2519 preempt_enable();
1da177e4 2520
808f5114 2521 spin_lock(&po->bind_lock);
ce06b03e 2522 unregister_prot_hook(sk, false);
66e56cd4
DB		 2523 packet_cached_dev_reset(po);
2524
160ff18a
BG		 2525 if (po->prot_hook.dev) {
2526 dev_put(po->prot_hook.dev);
2527 po->prot_hook.dev = NULL;
2528 }
808f5114 2529 spin_unlock(&po->bind_lock);
1da177e4 2530
1da177e4 2531 packet_flush_mclist(sk);
1da177e4 2532
9665d5d6
PS		 2533 if (po->rx_ring.pg_vec) {
2534 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2535 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2536 }
69e3c75f 2537
9665d5d6
PS		 2538 if (po->tx_ring.pg_vec) {
2539 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2540 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2541 }
1da177e4 2542
dc99f600
DM		 2543 fanout_release(sk);
2544
808f5114 2545 synchronize_net();
1da177e4
LT		 2546 /*
2547 * Now the socket is dead. No more input will appear.
2548 */
1da177e4
LT		 2549 sock_orphan(sk);
2550 sock->sk = NULL;
2551
2552 /* Purge queues */
2553
2554 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2555 sk_refcnt_debug_release(sk);
1da177e4
LT		 2556
2557 sock_put(sk);
2558 return 0;
2559}
2560
2561/*
2562 * Attach a packet hook.
2563 */
2564
0e11c91e 2565static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2566{
2567 struct packet_sock *po = pkt_sk(sk);
dc99f600 2568
aef950b4
WY		 2569 if (po->fanout) {
2570 if (dev)
2571 dev_put(dev);
2572
dc99f600 2573 return -EINVAL;
aef950b4 2574 }
1da177e4
LT		 2575
2576 lock_sock(sk);
2577
2578 spin_lock(&po->bind_lock);
ce06b03e 2579 unregister_prot_hook(sk, true);
66e56cd4 2580
1da177e4
LT		 2581 po->num = protocol;
2582 po->prot_hook.type = protocol;
160ff18a
BG		 2583 if (po->prot_hook.dev)
2584 dev_put(po->prot_hook.dev);
1da177e4 2585
66e56cd4 2586 po->prot_hook.dev = dev;
1da177e4
LT		 2587 po->ifindex = dev ? dev->ifindex : 0;
2588
66e56cd4
DB		 2589 packet_cached_dev_assign(po, dev);
2590
1da177e4
LT		 2591 if (protocol == 0)
2592 goto out_unlock;
2593
be85d4ad 2594 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2595 register_prot_hook(sk);
be85d4ad
UT		 2596 } else {
2597 sk->sk_err = ENETDOWN;
2598 if (!sock_flag(sk, SOCK_DEAD))
2599 sk->sk_error_report(sk);
1da177e4
LT		 2600 }
2601
2602out_unlock:
2603 spin_unlock(&po->bind_lock);
2604 release_sock(sk);
2605 return 0;
2606}
2607
2608/*
2609 * Bind a packet socket to a device
2610 */
2611
40d4e3df
ED		 2612static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2613 int addr_len)
1da177e4 2614{
40d4e3df 2615 struct sock *sk = sock->sk;
1da177e4
LT		 2616 char name[15];
2617 struct net_device *dev;
2618 int err = -ENODEV;
1ce4f28b 2619
1da177e4
LT		 2620 /*
2621 * Check legality
2622 */
1ce4f28b 2623
8ae55f04 2624 if (addr_len != sizeof(struct sockaddr))
1da177e4 2625 return -EINVAL;
40d4e3df 2626 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2627
3b1e0a65 2628 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2629 if (dev)
1da177e4 2630 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2631 return err;
2632}
1da177e4
LT		 2633
2634static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2635{
40d4e3df
ED		 2636 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2637 struct sock *sk = sock->sk;
1da177e4
LT		 2638 struct net_device *dev = NULL;
2639 int err;
2640
2641
2642 /*
2643 * Check legality
2644 */
1ce4f28b 2645
1da177e4
LT		 2646 if (addr_len < sizeof(struct sockaddr_ll))
2647 return -EINVAL;
2648 if (sll->sll_family != AF_PACKET)
2649 return -EINVAL;
2650
2651 if (sll->sll_ifindex) {
2652 err = -ENODEV;
3b1e0a65 2653 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2654 if (dev == NULL)
2655 goto out;
2656 }
2657 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2658
2659out:
2660 return err;
2661}
2662
2663static struct proto packet_proto = {
2664 .name = "PACKET",
2665 .owner = THIS_MODULE,
2666 .obj_size = sizeof(struct packet_sock),
2667};
2668
2669/*
1ce4f28b 2670 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2671 */
2672
3f378b68
EP		 2673static int packet_create(struct net *net, struct socket *sock, int protocol,
2674 int kern)
1da177e4
LT		 2675{
2676 struct sock *sk;
2677 struct packet_sock *po;
0e11c91e 2678 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2679 int err;
2680
df008c91 2681 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2682 return -EPERM;
be02097c
DM		 2683 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2684 sock->type != SOCK_PACKET)
1da177e4
LT		 2685 return -ESOCKTNOSUPPORT;
2686
2687 sock->state = SS_UNCONNECTED;
2688
2689 err = -ENOBUFS;
6257ff21 2690 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2691 if (sk == NULL)
2692 goto out;
2693
2694 sock->ops = &packet_ops;
1da177e4
LT		 2695 if (sock->type == SOCK_PACKET)
2696 sock->ops = &packet_ops_spkt;
be02097c 2697
1da177e4
LT		 2698 sock_init_data(sock, sk);
2699
2700 po = pkt_sk(sk);
2701 sk->sk_family = PF_PACKET;
0e11c91e 2702 po->num = proto;
d346a3fa 2703 po->xmit = dev_queue_xmit;
66e56cd4
DB		 2704
2705 packet_cached_dev_reset(po);
1da177e4
LT		 2706
2707 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2708 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2709
2710 /*
2711 * Attach a protocol block
2712 */
2713
2714 spin_lock_init(&po->bind_lock);
905db440 2715 mutex_init(&po->pg_vec_lock);
1da177e4 2716 po->prot_hook.func = packet_rcv;
be02097c 2717
1da177e4
LT		 2718 if (sock->type == SOCK_PACKET)
2719 po->prot_hook.func = packet_rcv_spkt;
be02097c 2720
1da177e4
LT		 2721 po->prot_hook.af_packet_priv = sk;
2722
0e11c91e
AV		 2723 if (proto) {
2724 po->prot_hook.type = proto;
ce06b03e 2725 register_prot_hook(sk);
1da177e4
LT		 2726 }
2727
0fa7fa98 2728 mutex_lock(&net->packet.sklist_lock);
808f5114 2729 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2730 mutex_unlock(&net->packet.sklist_lock);
2731
2732 preempt_disable();
3680453c 2733 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2734 preempt_enable();
808f5114 2735
40d4e3df 2736 return 0;
1da177e4
LT		 2737out:
2738 return err;
2739}
2740
2741/*
2742 * Pull a packet from our receive queue and hand it to the user.
2743 * If necessary we block.
2744 */
2745
2746static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2747 struct msghdr *msg, size_t len, int flags)
2748{
2749 struct sock *sk = sock->sk;
2750 struct sk_buff *skb;
2751 int copied, err;
bfd5f4a3 2752 int vnet_hdr_len = 0;
1da177e4
LT		 2753
2754 err = -EINVAL;
ed85b565 2755 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2756 goto out;
2757
2758#if 0
2759 /* What error should we return now? EUNATTACH? */
2760 if (pkt_sk(sk)->ifindex < 0)
2761 return -ENODEV;
2762#endif
2763
ed85b565 2764 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2765 err = sock_recv_errqueue(sk, msg, len,
2766 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2767 goto out;
2768 }
2769
1da177e4
LT		 2770 /*
2771 * Call the generic datagram receiver. This handles all sorts
2772 * of horrible races and re-entrancy so we can forget about it
2773 * in the protocol layers.
2774 *
2775 * Now it will return ENETDOWN, if device have just gone down,
2776 * but then it will block.
2777 */
2778
40d4e3df 2779 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2780
2781 /*
1ce4f28b 2782 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2783 * handles the blocking we don't see and worry about blocking
2784 * retries.
2785 */
2786
8ae55f04 2787 if (skb == NULL)
1da177e4
LT		 2788 goto out;
2789
bfd5f4a3
SS		 2790 if (pkt_sk(sk)->has_vnet_hdr) {
2791 struct virtio_net_hdr vnet_hdr = { 0 };
2792
2793 err = -EINVAL;
2794 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2795 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2796 goto out_free;
2797
1f18b717
MK		 2798 len -= vnet_hdr_len;
2799
bfd5f4a3
SS		 2800 if (skb_is_gso(skb)) {
2801 struct skb_shared_info *sinfo = skb_shinfo(skb);
2802
2803 /* This is a hint as to how much should be linear. */
2804 vnet_hdr.hdr_len = skb_headlen(skb);
2805 vnet_hdr.gso_size = sinfo->gso_size;
2806 if (sinfo->gso_type & SKB_GSO_TCPV4)
2807 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2808 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2809 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2810 else if (sinfo->gso_type & SKB_GSO_UDP)
2811 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2812 else if (sinfo->gso_type & SKB_GSO_FCOE)
2813 goto out_free;
2814 else
2815 BUG();
2816 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2817 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2818 } else
2819 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2820
2821 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2822 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2823 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2824 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2825 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2826 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2827 } /* else everything is zero */
2828
2829 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2830 vnet_hdr_len);
2831 if (err < 0)
2832 goto out_free;
2833 }
2834
f3d33426
HFS		 2835 /* You lose any data beyond the buffer you gave. If it worries
2836 * a user program they can ask the device for its MTU
2837 * anyway.
1da177e4 2838 */
1da177e4 2839 copied = skb->len;
40d4e3df
ED		 2840 if (copied > len) {
2841 copied = len;
2842 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2843 }
2844
2845 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2846 if (err)
2847 goto out_free;
2848
3b885787 2849 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2850
f3d33426
HFS		 2851 if (msg->msg_name) {
2852 /* If the address length field is there to be filled
2853 * in, we fill it in now.
2854 */
2855 if (sock->type == SOCK_PACKET) {
2856 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2857 } else {
2858 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2859 msg->msg_namelen = sll->sll_halen +
2860 offsetof(struct sockaddr_ll, sll_addr);
2861 }
ffbc6111
HX		 2862 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2863 msg->msg_namelen);
f3d33426 2864 }
1da177e4 2865
8dc41944 2866 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2867 struct tpacket_auxdata aux;
2868
2869 aux.tp_status = TP_STATUS_USER;
2870 if (skb->ip_summed == CHECKSUM_PARTIAL)
2871 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2872 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2873 aux.tp_snaplen = skb->len;
2874 aux.tp_mac = 0;
bbe735e4 2875 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2876 if (vlan_tx_tag_present(skb)) {
2877 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2878 aux.tp_status |= TP_STATUS_VLAN_VALID;
2879 } else {
2880 aux.tp_vlan_tci = 0;
2881 }
13fcb7bd 2882 aux.tp_padding = 0;
ffbc6111 2883 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2884 }
2885
1da177e4
LT		 2886 /*
2887 * Free or return the buffer as appropriate. Again this
2888 * hides all the races and re-entrancy issues from us.
2889 */
bfd5f4a3 2890 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2891
2892out_free:
2893 skb_free_datagram(sk, skb);
2894out:
2895 return err;
2896}
2897
1da177e4
LT		 2898static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2899 int *uaddr_len, int peer)
2900{
2901 struct net_device *dev;
2902 struct sock *sk = sock->sk;
2903
2904 if (peer)
2905 return -EOPNOTSUPP;
2906
2907 uaddr->sa_family = AF_PACKET;
2dc85bf3 2908 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 2909 rcu_read_lock();
2910 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2911 if (dev)
2dc85bf3 2912 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 2913 rcu_read_unlock();
1da177e4
LT		 2914 *uaddr_len = sizeof(*uaddr);
2915
2916 return 0;
2917}
1da177e4
LT		 2918
2919static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2920 int *uaddr_len, int peer)
2921{
2922 struct net_device *dev;
2923 struct sock *sk = sock->sk;
2924 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2925 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2926
2927 if (peer)
2928 return -EOPNOTSUPP;
2929
2930 sll->sll_family = AF_PACKET;
2931 sll->sll_ifindex = po->ifindex;
2932 sll->sll_protocol = po->num;
67286640 2933 sll->sll_pkttype = 0;
654d1f8a
ED		 2934 rcu_read_lock();
2935 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2936 if (dev) {
2937 sll->sll_hatype = dev->type;
2938 sll->sll_halen = dev->addr_len;
2939 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2940 } else {
2941 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2942 sll->sll_halen = 0;
2943 }
654d1f8a 2944 rcu_read_unlock();
0fb375fb 2945 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2946
2947 return 0;
2948}
2949
2aeb0b88
WC		 2950static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2951 int what)
1da177e4
LT		 2952{
2953 switch (i->type) {
2954 case PACKET_MR_MULTICAST:
1162563f
JP		 2955 if (i->alen != dev->addr_len)
2956 return -EINVAL;
1da177e4 2957 if (what > 0)
22bedad3 2958 return dev_mc_add(dev, i->addr);
1da177e4 2959 else
22bedad3 2960 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2961 break;
2962 case PACKET_MR_PROMISC:
2aeb0b88 2963 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2964 break;
2965 case PACKET_MR_ALLMULTI:
2aeb0b88 2966 return dev_set_allmulti(dev, what);
1da177e4 2967 break;
d95ed927 2968 case PACKET_MR_UNICAST:
1162563f
JP		 2969 if (i->alen != dev->addr_len)
2970 return -EINVAL;
d95ed927 2971 if (what > 0)
a748ee24 2972 return dev_uc_add(dev, i->addr);
d95ed927 2973 else
a748ee24 2974 return dev_uc_del(dev, i->addr);
d95ed927 2975 break;
40d4e3df
ED		 2976 default:
2977 break;
1da177e4 2978 }
2aeb0b88 2979 return 0;
1da177e4
LT		 2980}
2981
2982static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2983{
40d4e3df 2984 for ( ; i; i = i->next) {
1da177e4
LT		 2985 if (i->ifindex == dev->ifindex)
2986 packet_dev_mc(dev, i, what);
2987 }
2988}
2989
0fb375fb 2990static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2991{
2992 struct packet_sock *po = pkt_sk(sk);
2993 struct packet_mclist *ml, *i;
2994 struct net_device *dev;
2995 int err;
2996
2997 rtnl_lock();
2998
2999 err = -ENODEV;
3b1e0a65 3000 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3001 if (!dev)
3002 goto done;
3003
3004 err = -EINVAL;
1162563f 3005 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3006 goto done;
3007
3008 err = -ENOBUFS;
8b3a7005 3009 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3010 if (i == NULL)
3011 goto done;
3012
3013 err = 0;
3014 for (ml = po->mclist; ml; ml = ml->next) {
3015 if (ml->ifindex == mreq->mr_ifindex &&
3016 ml->type == mreq->mr_type &&
3017 ml->alen == mreq->mr_alen &&
3018 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3019 ml->count++;
3020 /* Free the new element ... */
3021 kfree(i);
3022 goto done;
3023 }
3024 }
3025
3026 i->type = mreq->mr_type;
3027 i->ifindex = mreq->mr_ifindex;
3028 i->alen = mreq->mr_alen;
3029 memcpy(i->addr, mreq->mr_address, i->alen);
3030 i->count = 1;
3031 i->next = po->mclist;
3032 po->mclist = i;
2aeb0b88
WC		 3033 err = packet_dev_mc(dev, i, 1);
3034 if (err) {
3035 po->mclist = i->next;
3036 kfree(i);
3037 }
1da177e4
LT		 3038
3039done:
3040 rtnl_unlock();
3041 return err;
3042}
3043
0fb375fb 3044static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3045{
3046 struct packet_mclist *ml, **mlp;
3047
3048 rtnl_lock();
3049
3050 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3051 if (ml->ifindex == mreq->mr_ifindex &&
3052 ml->type == mreq->mr_type &&
3053 ml->alen == mreq->mr_alen &&
3054 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3055 if (--ml->count == 0) {
3056 struct net_device *dev;
3057 *mlp = ml->next;
ad959e76
ED		 3058 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3059 if (dev)
1da177e4 3060 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3061 kfree(ml);
3062 }
3063 rtnl_unlock();
3064 return 0;
3065 }
3066 }
3067 rtnl_unlock();
3068 return -EADDRNOTAVAIL;
3069}
3070
3071static void packet_flush_mclist(struct sock *sk)
3072{
3073 struct packet_sock *po = pkt_sk(sk);
3074 struct packet_mclist *ml;
3075
3076 if (!po->mclist)
3077 return;
3078
3079 rtnl_lock();
3080 while ((ml = po->mclist) != NULL) {
3081 struct net_device *dev;
3082
3083 po->mclist = ml->next;
ad959e76
ED		 3084 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3085 if (dev != NULL)
1da177e4 3086 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3087 kfree(ml);
3088 }
3089 rtnl_unlock();
3090}
1da177e4
LT		 3091
3092static int
b7058842 3093packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3094{
3095 struct sock *sk = sock->sk;
8dc41944 3096 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3097 int ret;
3098
3099 if (level != SOL_PACKET)
3100 return -ENOPROTOOPT;
3101
69e3c75f 3102 switch (optname) {
1ce4f28b 3103 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3104 case PACKET_DROP_MEMBERSHIP:
3105 {
0fb375fb
EB		 3106 struct packet_mreq_max mreq;
3107 int len = optlen;
3108 memset(&mreq, 0, sizeof(mreq));
3109 if (len < sizeof(struct packet_mreq))
1da177e4 3110 return -EINVAL;
0fb375fb
EB		 3111 if (len > sizeof(mreq))
3112 len = sizeof(mreq);
40d4e3df 3113 if (copy_from_user(&mreq, optval, len))
1da177e4 3114 return -EFAULT;
0fb375fb
EB		 3115 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3116 return -EINVAL;
1da177e4
LT		 3117 if (optname == PACKET_ADD_MEMBERSHIP)
3118 ret = packet_mc_add(sk, &mreq);
3119 else
3120 ret = packet_mc_drop(sk, &mreq);
3121 return ret;
3122 }
a2efcfa0 3123
1da177e4 3124 case PACKET_RX_RING:
69e3c75f 3125 case PACKET_TX_RING:
1da177e4 3126 {
f6fb8f10 3127 union tpacket_req_u req_u;
3128 int len;
1da177e4 3129
f6fb8f10 3130 switch (po->tp_version) {
3131 case TPACKET_V1:
3132 case TPACKET_V2:
3133 len = sizeof(req_u.req);
3134 break;
3135 case TPACKET_V3:
3136 default:
3137 len = sizeof(req_u.req3);
3138 break;
3139 }
3140 if (optlen < len)
1da177e4 3141 return -EINVAL;
bfd5f4a3
SS		 3142 if (pkt_sk(sk)->has_vnet_hdr)
3143 return -EINVAL;
f6fb8f10 3144 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3145 return -EFAULT;
f6fb8f10 3146 return packet_set_ring(sk, &req_u, 0,
3147 optname == PACKET_TX_RING);
1da177e4
LT		 3148 }
3149 case PACKET_COPY_THRESH:
3150 {
3151 int val;
3152
40d4e3df 3153 if (optlen != sizeof(val))
1da177e4 3154 return -EINVAL;
40d4e3df 3155 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3156 return -EFAULT;
3157
3158 pkt_sk(sk)->copy_thresh = val;
3159 return 0;
3160 }
bbd6ef87
PM		 3161 case PACKET_VERSION:
3162 {
3163 int val;
3164
3165 if (optlen != sizeof(val))
3166 return -EINVAL;
69e3c75f 3167 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3168 return -EBUSY;
3169 if (copy_from_user(&val, optval, sizeof(val)))
3170 return -EFAULT;
3171 switch (val) {
3172 case TPACKET_V1:
3173 case TPACKET_V2:
f6fb8f10 3174 case TPACKET_V3:
bbd6ef87
PM		 3175 po->tp_version = val;
3176 return 0;
3177 default:
3178 return -EINVAL;
3179 }
3180 }
8913336a
PM		 3181 case PACKET_RESERVE:
3182 {
3183 unsigned int val;
3184
3185 if (optlen != sizeof(val))
3186 return -EINVAL;
69e3c75f 3187 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3188 return -EBUSY;
3189 if (copy_from_user(&val, optval, sizeof(val)))
3190 return -EFAULT;
3191 po->tp_reserve = val;
3192 return 0;
3193 }
69e3c75f
JB		 3194 case PACKET_LOSS:
3195 {
3196 unsigned int val;
3197
3198 if (optlen != sizeof(val))
3199 return -EINVAL;
3200 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3201 return -EBUSY;
3202 if (copy_from_user(&val, optval, sizeof(val)))
3203 return -EFAULT;
3204 po->tp_loss = !!val;
3205 return 0;
3206 }
8dc41944
HX		 3207 case PACKET_AUXDATA:
3208 {
3209 int val;
3210
3211 if (optlen < sizeof(val))
3212 return -EINVAL;
3213 if (copy_from_user(&val, optval, sizeof(val)))
3214 return -EFAULT;
3215
3216 po->auxdata = !!val;
3217 return 0;
3218 }
80feaacb
PWJ		 3219 case PACKET_ORIGDEV:
3220 {
3221 int val;
3222
3223 if (optlen < sizeof(val))
3224 return -EINVAL;
3225 if (copy_from_user(&val, optval, sizeof(val)))
3226 return -EFAULT;
3227
3228 po->origdev = !!val;
3229 return 0;
3230 }
bfd5f4a3
SS		 3231 case PACKET_VNET_HDR:
3232 {
3233 int val;
3234
3235 if (sock->type != SOCK_RAW)
3236 return -EINVAL;
3237 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3238 return -EBUSY;
3239 if (optlen < sizeof(val))
3240 return -EINVAL;
3241 if (copy_from_user(&val, optval, sizeof(val)))
3242 return -EFAULT;
3243
3244 po->has_vnet_hdr = !!val;
3245 return 0;
3246 }
614f60fa
SM		 3247 case PACKET_TIMESTAMP:
3248 {
3249 int val;
3250
3251 if (optlen != sizeof(val))
3252 return -EINVAL;
3253 if (copy_from_user(&val, optval, sizeof(val)))
3254 return -EFAULT;
3255
3256 po->tp_tstamp = val;
3257 return 0;
3258 }
dc99f600
DM		 3259 case PACKET_FANOUT:
3260 {
3261 int val;
3262
3263 if (optlen != sizeof(val))
3264 return -EINVAL;
3265 if (copy_from_user(&val, optval, sizeof(val)))
3266 return -EFAULT;
3267
3268 return fanout_add(sk, val & 0xffff, val >> 16);
3269 }
5920cd3a
PC		 3270 case PACKET_TX_HAS_OFF:
3271 {
3272 unsigned int val;
3273
3274 if (optlen != sizeof(val))
3275 return -EINVAL;
3276 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3277 return -EBUSY;
3278 if (copy_from_user(&val, optval, sizeof(val)))
3279 return -EFAULT;
3280 po->tp_tx_has_off = !!val;
3281 return 0;
3282 }
d346a3fa
DB		 3283 case PACKET_QDISC_BYPASS:
3284 {
3285 int val;
3286
3287 if (optlen != sizeof(val))
3288 return -EINVAL;
3289 if (copy_from_user(&val, optval, sizeof(val)))
3290 return -EFAULT;
3291
3292 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3293 return 0;
3294 }
1da177e4
LT		 3295 default:
3296 return -ENOPROTOOPT;
3297 }
3298}
3299
3300static int packet_getsockopt(struct socket *sock, int level, int optname,
3301 char __user *optval, int __user *optlen)
3302{
3303 int len;
c06fff6e 3304 int val, lv = sizeof(val);
1da177e4
LT		 3305 struct sock *sk = sock->sk;
3306 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3307 void *data = &val;
ee80fbf3 3308 union tpacket_stats_u st;
1da177e4
LT		 3309
3310 if (level != SOL_PACKET)
3311 return -ENOPROTOOPT;
3312
8ae55f04
KK		 3313 if (get_user(len, optlen))
3314 return -EFAULT;
1da177e4
LT		 3315
3316 if (len < 0)
3317 return -EINVAL;
1ce4f28b 3318
69e3c75f 3319 switch (optname) {
1da177e4 3320 case PACKET_STATISTICS:
1da177e4 3321 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3322 memcpy(&st, &po->stats, sizeof(st));
3323 memset(&po->stats, 0, sizeof(po->stats));
3324 spin_unlock_bh(&sk->sk_receive_queue.lock);
3325
f6fb8f10 3326 if (po->tp_version == TPACKET_V3) {
c06fff6e 3327 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3328 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3329 data = &st.stats3;
f6fb8f10 3330 } else {
c06fff6e 3331 lv = sizeof(struct tpacket_stats);
8bcdeaff 3332 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3333 data = &st.stats1;
f6fb8f10 3334 }
ee80fbf3 3335
8dc41944
HX		 3336 break;
3337 case PACKET_AUXDATA:
8dc41944 3338 val = po->auxdata;
80feaacb
PWJ		 3339 break;
3340 case PACKET_ORIGDEV:
80feaacb 3341 val = po->origdev;
bfd5f4a3
SS		 3342 break;
3343 case PACKET_VNET_HDR:
bfd5f4a3 3344 val = po->has_vnet_hdr;
1da177e4 3345 break;
bbd6ef87 3346 case PACKET_VERSION:
bbd6ef87 3347 val = po->tp_version;
bbd6ef87
PM		 3348 break;
3349 case PACKET_HDRLEN:
3350 if (len > sizeof(int))
3351 len = sizeof(int);
3352 if (copy_from_user(&val, optval, len))
3353 return -EFAULT;
3354 switch (val) {
3355 case TPACKET_V1:
3356 val = sizeof(struct tpacket_hdr);
3357 break;
3358 case TPACKET_V2:
3359 val = sizeof(struct tpacket2_hdr);
3360 break;
f6fb8f10 3361 case TPACKET_V3:
3362 val = sizeof(struct tpacket3_hdr);
3363 break;
bbd6ef87
PM		 3364 default:
3365 return -EINVAL;
3366 }
bbd6ef87 3367 break;
8913336a 3368 case PACKET_RESERVE:
8913336a 3369 val = po->tp_reserve;
8913336a 3370 break;
69e3c75f 3371 case PACKET_LOSS:
69e3c75f 3372 val = po->tp_loss;
69e3c75f 3373 break;
614f60fa 3374 case PACKET_TIMESTAMP:
614f60fa 3375 val = po->tp_tstamp;
614f60fa 3376 break;
dc99f600 3377 case PACKET_FANOUT:
dc99f600
DM		 3378 val = (po->fanout ?
3379 ((u32)po->fanout->id |
77f65ebd
WB		 3380 ((u32)po->fanout->type << 16) |
3381 ((u32)po->fanout->flags << 24)) :
dc99f600 3382 0);
dc99f600 3383 break;
5920cd3a
PC		 3384 case PACKET_TX_HAS_OFF:
3385 val = po->tp_tx_has_off;
3386 break;
d346a3fa
DB		 3387 case PACKET_QDISC_BYPASS:
3388 val = packet_use_direct_xmit(po);
3389 break;
1da177e4
LT		 3390 default:
3391 return -ENOPROTOOPT;
3392 }
3393
c06fff6e
ED		 3394 if (len > lv)
3395 len = lv;
8ae55f04
KK		 3396 if (put_user(len, optlen))
3397 return -EFAULT;
8dc41944
HX		 3398 if (copy_to_user(optval, data, len))
3399 return -EFAULT;
8ae55f04 3400 return 0;
1da177e4
LT		 3401}
3402
3403
351638e7
JP		 3404static int packet_notifier(struct notifier_block *this,
3405 unsigned long msg, void *ptr)
1da177e4
LT		 3406{
3407 struct sock *sk;
351638e7 3408 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3409 struct net *net = dev_net(dev);
1da177e4 3410
808f5114 3411 rcu_read_lock();
b67bfe0d 3412 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3413 struct packet_sock *po = pkt_sk(sk);
3414
3415 switch (msg) {
3416 case NETDEV_UNREGISTER:
1da177e4
LT		 3417 if (po->mclist)
3418 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3419 /* fallthrough */
3420
1da177e4
LT		 3421 case NETDEV_DOWN:
3422 if (dev->ifindex == po->ifindex) {
3423 spin_lock(&po->bind_lock);
3424 if (po->running) {
ce06b03e 3425 __unregister_prot_hook(sk, false);
1da177e4
LT		 3426 sk->sk_err = ENETDOWN;
3427 if (!sock_flag(sk, SOCK_DEAD))
3428 sk->sk_error_report(sk);
3429 }
3430 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3431 packet_cached_dev_reset(po);
1da177e4 3432 po->ifindex = -1;
160ff18a
BG		 3433 if (po->prot_hook.dev)
3434 dev_put(po->prot_hook.dev);
1da177e4
LT		 3435 po->prot_hook.dev = NULL;
3436 }
3437 spin_unlock(&po->bind_lock);
3438 }
3439 break;
3440 case NETDEV_UP:
808f5114 3441 if (dev->ifindex == po->ifindex) {
3442 spin_lock(&po->bind_lock);
ce06b03e
DM		 3443 if (po->num)
3444 register_prot_hook(sk);
808f5114 3445 spin_unlock(&po->bind_lock);
1da177e4 3446 }
1da177e4
LT		 3447 break;
3448 }
3449 }
808f5114 3450 rcu_read_unlock();
1da177e4
LT		 3451 return NOTIFY_DONE;
3452}
3453
3454
3455static int packet_ioctl(struct socket *sock, unsigned int cmd,
3456 unsigned long arg)
3457{
3458 struct sock *sk = sock->sk;
3459
69e3c75f 3460 switch (cmd) {
40d4e3df
ED		 3461 case SIOCOUTQ:
3462 {
3463 int amount = sk_wmem_alloc_get(sk);
31e6d363 3464
40d4e3df
ED		 3465 return put_user(amount, (int __user *)arg);
3466 }
3467 case SIOCINQ:
3468 {
3469 struct sk_buff *skb;
3470 int amount = 0;
3471
3472 spin_lock_bh(&sk->sk_receive_queue.lock);
3473 skb = skb_peek(&sk->sk_receive_queue);
3474 if (skb)
3475 amount = skb->len;
3476 spin_unlock_bh(&sk->sk_receive_queue.lock);
3477 return put_user(amount, (int __user *)arg);
3478 }
3479 case SIOCGSTAMP:
3480 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3481 case SIOCGSTAMPNS:
3482 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3483
1da177e4 3484#ifdef CONFIG_INET
40d4e3df
ED		 3485 case SIOCADDRT:
3486 case SIOCDELRT:
3487 case SIOCDARP:
3488 case SIOCGARP:
3489 case SIOCSARP:
3490 case SIOCGIFADDR:
3491 case SIOCSIFADDR:
3492 case SIOCGIFBRDADDR:
3493 case SIOCSIFBRDADDR:
3494 case SIOCGIFNETMASK:
3495 case SIOCSIFNETMASK:
3496 case SIOCGIFDSTADDR:
3497 case SIOCSIFDSTADDR:
3498 case SIOCSIFFLAGS:
40d4e3df 3499 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3500#endif
3501
40d4e3df
ED		 3502 default:
3503 return -ENOIOCTLCMD;
1da177e4
LT		 3504 }
3505 return 0;
3506}
3507
40d4e3df 3508static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3509 poll_table *wait)
3510{
3511 struct sock *sk = sock->sk;
3512 struct packet_sock *po = pkt_sk(sk);
3513 unsigned int mask = datagram_poll(file, sock, wait);
3514
3515 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3516 if (po->rx_ring.pg_vec) {
f6fb8f10 3517 if (!packet_previous_rx_frame(po, &po->rx_ring,
3518 TP_STATUS_KERNEL))
1da177e4
LT		 3519 mask |= POLLIN | POLLRDNORM;
3520 }
3521 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3522 spin_lock_bh(&sk->sk_write_queue.lock);
3523 if (po->tx_ring.pg_vec) {
3524 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3525 mask |= POLLOUT | POLLWRNORM;
3526 }
3527 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3528 return mask;
3529}
3530
3531
3532/* Dirty? Well, I still did not learn better way to account
3533 * for user mmaps.
3534 */
3535
3536static void packet_mm_open(struct vm_area_struct *vma)
3537{
3538 struct file *file = vma->vm_file;
40d4e3df 3539 struct socket *sock = file->private_data;
1da177e4 3540 struct sock *sk = sock->sk;
1ce4f28b 3541
1da177e4
LT		 3542 if (sk)
3543 atomic_inc(&pkt_sk(sk)->mapped);
3544}
3545
3546static void packet_mm_close(struct vm_area_struct *vma)
3547{
3548 struct file *file = vma->vm_file;
40d4e3df 3549 struct socket *sock = file->private_data;
1da177e4 3550 struct sock *sk = sock->sk;
1ce4f28b 3551
1da177e4
LT		 3552 if (sk)
3553 atomic_dec(&pkt_sk(sk)->mapped);
3554}
3555
f0f37e2f 3556static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3557 .open = packet_mm_open,
3558 .close = packet_mm_close,
1da177e4
LT		 3559};
3560
0e3125c7
NH		 3561static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3562 unsigned int len)
1da177e4
LT		 3563{
3564 int i;
3565
4ebf0ae2 3566 for (i = 0; i < len; i++) {
0e3125c7 3567 if (likely(pg_vec[i].buffer)) {
c56b4d90 3568 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3569 vfree(pg_vec[i].buffer);
3570 else
3571 free_pages((unsigned long)pg_vec[i].buffer,
3572 order);
3573 pg_vec[i].buffer = NULL;
3574 }
1da177e4
LT		 3575 }
3576 kfree(pg_vec);
3577}
3578
eea49cc9 3579static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3580{
0e3125c7
NH		 3581 char *buffer = NULL;
3582 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3583 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3584
3585 buffer = (char *) __get_free_pages(gfp_flags, order);
3586
3587 if (buffer)
3588 return buffer;
3589
3590 /*
3591 * __get_free_pages failed, fall back to vmalloc
3592 */
bbce5a59 3593 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3594
0e3125c7
NH		 3595 if (buffer)
3596 return buffer;
3597
3598 /*
3599 * vmalloc failed, lets dig into swap here
3600 */
0e3125c7
NH		 3601 gfp_flags &= ~__GFP_NORETRY;
3602 buffer = (char *)__get_free_pages(gfp_flags, order);
3603 if (buffer)
3604 return buffer;
3605
3606 /*
3607 * complete and utter failure
3608 */
3609 return NULL;
4ebf0ae2
DM		 3610}
3611
0e3125c7 3612static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3613{
3614 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3615 struct pgv *pg_vec;
4ebf0ae2
DM		 3616 int i;
3617
0e3125c7 3618 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3619 if (unlikely(!pg_vec))
3620 goto out;
3621
3622 for (i = 0; i < block_nr; i++) {
c56b4d90 3623 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3624 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3625 goto out_free_pgvec;
3626 }
3627
3628out:
3629 return pg_vec;
3630
3631out_free_pgvec:
3632 free_pg_vec(pg_vec, order, block_nr);
3633 pg_vec = NULL;
3634 goto out;
3635}
1da177e4 3636
f6fb8f10 3637static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3638 int closing, int tx_ring)
1da177e4 3639{
0e3125c7 3640 struct pgv *pg_vec = NULL;
1da177e4 3641 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3642 int was_running, order = 0;
69e3c75f
JB		 3643 struct packet_ring_buffer *rb;
3644 struct sk_buff_head *rb_queue;
0e11c91e 3645 __be16 num;
f6fb8f10 3646 int err = -EINVAL;
3647 /* Added to avoid minimal code churn */
3648 struct tpacket_req *req = &req_u->req;
3649
3650 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3651 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3652 WARN(1, "Tx-ring is not supported.\n");
3653 goto out;
3654 }
1ce4f28b 3655
69e3c75f
JB		 3656 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3657 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3658
69e3c75f
JB		 3659 err = -EBUSY;
3660 if (!closing) {
3661 if (atomic_read(&po->mapped))
3662 goto out;
3663 if (atomic_read(&rb->pending))
3664 goto out;
3665 }
1da177e4 3666
69e3c75f
JB		 3667 if (req->tp_block_nr) {
3668 /* Sanity tests and some calculations */
3669 err = -EBUSY;
3670 if (unlikely(rb->pg_vec))
3671 goto out;
1da177e4 3672
bbd6ef87
PM		 3673 switch (po->tp_version) {
3674 case TPACKET_V1:
3675 po->tp_hdrlen = TPACKET_HDRLEN;
3676 break;
3677 case TPACKET_V2:
3678 po->tp_hdrlen = TPACKET2_HDRLEN;
3679 break;
f6fb8f10 3680 case TPACKET_V3:
3681 po->tp_hdrlen = TPACKET3_HDRLEN;
3682 break;
bbd6ef87
PM		 3683 }
3684
69e3c75f 3685 err = -EINVAL;
4ebf0ae2 3686 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3687 goto out;
4ebf0ae2 3688 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3689 goto out;
8913336a 3690 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3691 po->tp_reserve))
3692 goto out;
4ebf0ae2 3693 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3694 goto out;
1da177e4 3695
69e3c75f
JB		 3696 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3697 if (unlikely(rb->frames_per_block <= 0))
3698 goto out;
3699 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3700 req->tp_frame_nr))
3701 goto out;
1da177e4
LT		 3702
3703 err = -ENOMEM;
4ebf0ae2
DM		 3704 order = get_order(req->tp_block_size);
3705 pg_vec = alloc_pg_vec(req, order);
3706 if (unlikely(!pg_vec))
1da177e4 3707 goto out;
f6fb8f10 3708 switch (po->tp_version) {
3709 case TPACKET_V3:
3710 /* Transmit path is not supported. We checked
3711 * it above but just being paranoid
3712 */
3713 if (!tx_ring)
3714 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3715 break;
3716 default:
3717 break;
3718 }
69e3c75f
JB		 3719 }
3720 /* Done */
3721 else {
3722 err = -EINVAL;
4ebf0ae2 3723 if (unlikely(req->tp_frame_nr))
69e3c75f 3724 goto out;
1da177e4
LT		 3725 }
3726
3727 lock_sock(sk);
3728
3729 /* Detach socket from network */
3730 spin_lock(&po->bind_lock);
3731 was_running = po->running;
3732 num = po->num;
3733 if (was_running) {
1da177e4 3734 po->num = 0;
ce06b03e 3735 __unregister_prot_hook(sk, false);
1da177e4
LT		 3736 }
3737 spin_unlock(&po->bind_lock);
1ce4f28b 3738
1da177e4
LT		 3739 synchronize_net();
3740
3741 err = -EBUSY;
905db440 3742 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3743 if (closing || atomic_read(&po->mapped) == 0) {
3744 err = 0;
69e3c75f 3745 spin_lock_bh(&rb_queue->lock);
c053fd96 3746 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3747 rb->frame_max = (req->tp_frame_nr - 1);
3748 rb->head = 0;
3749 rb->frame_size = req->tp_frame_size;
3750 spin_unlock_bh(&rb_queue->lock);
3751
c053fd96
CG		 3752 swap(rb->pg_vec_order, order);
3753 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3754
3755 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3756 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3757 tpacket_rcv : packet_rcv;
3758 skb_queue_purge(rb_queue);
1da177e4 3759 if (atomic_read(&po->mapped))
40d4e3df
ED		 3760 pr_err("packet_mmap: vma is busy: %d\n",
3761 atomic_read(&po->mapped));
1da177e4 3762 }
905db440 3763 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3764
3765 spin_lock(&po->bind_lock);
ce06b03e 3766 if (was_running) {
1da177e4 3767 po->num = num;
ce06b03e 3768 register_prot_hook(sk);
1da177e4
LT		 3769 }
3770 spin_unlock(&po->bind_lock);
f6fb8f10 3771 if (closing && (po->tp_version > TPACKET_V2)) {
3772 /* Because we don't support block-based V3 on tx-ring */
3773 if (!tx_ring)
3774 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3775 }
1da177e4
LT		 3776 release_sock(sk);
3777
1da177e4
LT		 3778 if (pg_vec)
3779 free_pg_vec(pg_vec, order, req->tp_block_nr);
3780out:
3781 return err;
3782}
3783
69e3c75f
JB		 3784static int packet_mmap(struct file *file, struct socket *sock,
3785 struct vm_area_struct *vma)
1da177e4
LT		 3786{
3787 struct sock *sk = sock->sk;
3788 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3789 unsigned long size, expected_size;
3790 struct packet_ring_buffer *rb;
1da177e4
LT		 3791 unsigned long start;
3792 int err = -EINVAL;
3793 int i;
3794
3795 if (vma->vm_pgoff)
3796 return -EINVAL;
3797
905db440 3798 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3799
3800 expected_size = 0;
3801 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3802 if (rb->pg_vec) {
3803 expected_size += rb->pg_vec_len
3804 * rb->pg_vec_pages
3805 * PAGE_SIZE;
3806 }
3807 }
3808
3809 if (expected_size == 0)
1da177e4 3810 goto out;
69e3c75f
JB		 3811
3812 size = vma->vm_end - vma->vm_start;
3813 if (size != expected_size)
1da177e4
LT		 3814 goto out;
3815
1da177e4 3816 start = vma->vm_start;
69e3c75f
JB		 3817 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3818 if (rb->pg_vec == NULL)
3819 continue;
3820
3821 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3822 struct page *page;
3823 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3824 int pg_num;
3825
c56b4d90
CG		 3826 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3827 page = pgv_to_page(kaddr);
69e3c75f
JB		 3828 err = vm_insert_page(vma, start, page);
3829 if (unlikely(err))
3830 goto out;
3831 start += PAGE_SIZE;
0e3125c7 3832 kaddr += PAGE_SIZE;
69e3c75f 3833 }
4ebf0ae2 3834 }
1da177e4 3835 }
69e3c75f 3836
4ebf0ae2 3837 atomic_inc(&po->mapped);
1da177e4
LT		 3838 vma->vm_ops = &packet_mmap_ops;
3839 err = 0;
3840
3841out:
905db440 3842 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3843 return err;
3844}
1da177e4 3845
90ddc4f0 3846static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3847 .family = PF_PACKET,
3848 .owner = THIS_MODULE,
3849 .release = packet_release,
3850 .bind = packet_bind_spkt,
3851 .connect = sock_no_connect,
3852 .socketpair = sock_no_socketpair,
3853 .accept = sock_no_accept,
3854 .getname = packet_getname_spkt,
3855 .poll = datagram_poll,
3856 .ioctl = packet_ioctl,
3857 .listen = sock_no_listen,
3858 .shutdown = sock_no_shutdown,
3859 .setsockopt = sock_no_setsockopt,
3860 .getsockopt = sock_no_getsockopt,
3861 .sendmsg = packet_sendmsg_spkt,
3862 .recvmsg = packet_recvmsg,
3863 .mmap = sock_no_mmap,
3864 .sendpage = sock_no_sendpage,
3865};
1da177e4 3866
90ddc4f0 3867static const struct proto_ops packet_ops = {
1da177e4
LT		 3868 .family = PF_PACKET,
3869 .owner = THIS_MODULE,
3870 .release = packet_release,
3871 .bind = packet_bind,
3872 .connect = sock_no_connect,
3873 .socketpair = sock_no_socketpair,
3874 .accept = sock_no_accept,
1ce4f28b 3875 .getname = packet_getname,
1da177e4
LT		 3876 .poll = packet_poll,
3877 .ioctl = packet_ioctl,
3878 .listen = sock_no_listen,
3879 .shutdown = sock_no_shutdown,
3880 .setsockopt = packet_setsockopt,
3881 .getsockopt = packet_getsockopt,
3882 .sendmsg = packet_sendmsg,
3883 .recvmsg = packet_recvmsg,
3884 .mmap = packet_mmap,
3885 .sendpage = sock_no_sendpage,
3886};
3887
ec1b4cf7 3888static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3889 .family = PF_PACKET,
3890 .create = packet_create,
3891 .owner = THIS_MODULE,
3892};
3893
3894static struct notifier_block packet_netdev_notifier = {
40d4e3df 3895 .notifier_call = packet_notifier,
1da177e4
LT		 3896};
3897
3898#ifdef CONFIG_PROC_FS
1da177e4
LT		 3899
3900static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3901 __acquires(RCU)
1da177e4 3902{
e372c414 3903 struct net *net = seq_file_net(seq);
808f5114 3904
3905 rcu_read_lock();
3906 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3907}
3908
3909static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3910{
1bf40954 3911 struct net *net = seq_file_net(seq);
808f5114 3912 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3913}
3914
3915static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3916 __releases(RCU)
1da177e4 3917{
808f5114 3918 rcu_read_unlock();
1da177e4
LT		 3919}
3920
1ce4f28b 3921static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3922{
3923 if (v == SEQ_START_TOKEN)
3924 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3925 else {
b7ceabd9 3926 struct sock *s = sk_entry(v);
1da177e4
LT		 3927 const struct packet_sock *po = pkt_sk(s);
3928
3929 seq_printf(seq,
71338aa7 3930 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3931 s,
3932 atomic_read(&s->sk_refcnt),
3933 s->sk_type,
3934 ntohs(po->num),
3935 po->ifindex,
3936 po->running,
3937 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 3938 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 3939 sock_i_ino(s));
1da177e4
LT		 3940 }
3941
3942 return 0;
3943}
3944
56b3d975 3945static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3946 .start = packet_seq_start,
3947 .next = packet_seq_next,
3948 .stop = packet_seq_stop,
3949 .show = packet_seq_show,
3950};
3951
3952static int packet_seq_open(struct inode *inode, struct file *file)
3953{
e372c414
DL		 3954 return seq_open_net(inode, file, &packet_seq_ops,
3955 sizeof(struct seq_net_private));
1da177e4
LT		 3956}
3957
da7071d7 3958static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3959 .owner = THIS_MODULE,
3960 .open = packet_seq_open,
3961 .read = seq_read,
3962 .llseek = seq_lseek,
e372c414 3963 .release = seq_release_net,
1da177e4
LT		 3964};
3965
3966#endif
3967
2c8c1e72 3968static int __net_init packet_net_init(struct net *net)
d12d01d6 3969{
0fa7fa98 3970 mutex_init(&net->packet.sklist_lock);
2aaef4e4 3971 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 3972
d4beaa66 3973 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 3974 return -ENOMEM;
3975
3976 return 0;
3977}
3978
2c8c1e72 3979static void __net_exit packet_net_exit(struct net *net)
d12d01d6 3980{
ece31ffd 3981 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 3982}
3983
3984static struct pernet_operations packet_net_ops = {
3985 .init = packet_net_init,
3986 .exit = packet_net_exit,
3987};
3988
3989
1da177e4
LT		 3990static void __exit packet_exit(void)
3991{
1da177e4 3992 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3993 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3994 sock_unregister(PF_PACKET);
3995 proto_unregister(&packet_proto);
3996}
3997
3998static int __init packet_init(void)
3999{
4000 int rc = proto_register(&packet_proto, 0);
4001
4002 if (rc != 0)
4003 goto out;
4004
4005 sock_register(&packet_family_ops);
d12d01d6 4006 register_pernet_subsys(&packet_net_ops);
1da177e4 4007 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4008out:
4009 return rc;
4010}
4011
4012module_init(packet_init);
4013module_exit(packet_exit);
4014MODULE_LICENSE("GPL");
4015MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 0.996285 seconds and 5 git commands to generate.