2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * Implementation of the Transmission Control Protocol(TCP).
8 * Version: $Id: tcp.c,v 1.216 2002/02/01 22:01:04 davem Exp $
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 * Corey Minyard <wf-rch!minyard@relay.EU.net>
14 * Florian La Roche, <flla@stud.uni-sb.de>
15 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
16 * Linus Torvalds, <torvalds@cs.helsinki.fi>
17 * Alan Cox, <gw4pts@gw4pts.ampr.org>
18 * Matthew Dillon, <dillon@apollo.west.oic.com>
19 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
20 * Jorge Cwik, <jorge@laser.satlink.net>
23 * Alan Cox : Numerous verify_area() calls
24 * Alan Cox : Set the ACK bit on a reset
25 * Alan Cox : Stopped it crashing if it closed while
26 * sk->inuse=1 and was trying to connect
28 * Alan Cox : All icmp error handling was broken
29 * pointers passed where wrong and the
30 * socket was looked up backwards. Nobody
31 * tested any icmp error code obviously.
32 * Alan Cox : tcp_err() now handled properly. It
33 * wakes people on errors. poll
34 * behaves and the icmp error race
35 * has gone by moving it into sock.c
36 * Alan Cox : tcp_send_reset() fixed to work for
37 * everything not just packets for
39 * Alan Cox : tcp option processing.
40 * Alan Cox : Reset tweaked (still not 100%) [Had
42 * Herp Rosmanith : More reset fixes
43 * Alan Cox : No longer acks invalid rst frames.
44 * Acking any kind of RST is right out.
45 * Alan Cox : Sets an ignore me flag on an rst
46 * receive otherwise odd bits of prattle
48 * Alan Cox : Fixed another acking RST frame bug.
49 * Should stop LAN workplace lockups.
50 * Alan Cox : Some tidyups using the new skb list
52 * Alan Cox : sk->keepopen now seems to work
53 * Alan Cox : Pulls options out correctly on accepts
54 * Alan Cox : Fixed assorted sk->rqueue->next errors
55 * Alan Cox : PSH doesn't end a TCP read. Switched a
57 * Alan Cox : Tidied tcp_data to avoid a potential
59 * Alan Cox : Added some better commenting, as the
60 * tcp is hard to follow
61 * Alan Cox : Removed incorrect check for 20 * psh
62 * Michael O'Reilly : ack < copied bug fix.
63 * Johannes Stille : Misc tcp fixes (not all in yet).
64 * Alan Cox : FIN with no memory -> CRASH
65 * Alan Cox : Added socket option proto entries.
66 * Also added awareness of them to accept.
67 * Alan Cox : Added TCP options (SOL_TCP)
68 * Alan Cox : Switched wakeup calls to callbacks,
69 * so the kernel can layer network
71 * Alan Cox : Use ip_tos/ip_ttl settings.
72 * Alan Cox : Handle FIN (more) properly (we hope).
73 * Alan Cox : RST frames sent on unsynchronised
75 * Alan Cox : Put in missing check for SYN bit.
76 * Alan Cox : Added tcp_select_window() aka NET2E
77 * window non shrink trick.
78 * Alan Cox : Added a couple of small NET2E timer
80 * Charles Hedrick : TCP fixes
81 * Toomas Tamm : TCP window fixes
82 * Alan Cox : Small URG fix to rlogin ^C ack fight
83 * Charles Hedrick : Rewrote most of it to actually work
84 * Linus : Rewrote tcp_read() and URG handling
86 * Gerhard Koerting: Fixed some missing timer handling
87 * Matthew Dillon : Reworked TCP machine states as per RFC
88 * Gerhard Koerting: PC/TCP workarounds
89 * Adam Caldwell : Assorted timer/timing errors
90 * Matthew Dillon : Fixed another RST bug
91 * Alan Cox : Move to kernel side addressing changes.
92 * Alan Cox : Beginning work on TCP fastpathing
94 * Arnt Gulbrandsen: Turbocharged tcp_check() routine.
95 * Alan Cox : TCP fast path debugging
96 * Alan Cox : Window clamping
97 * Michael Riepe : Bug in tcp_check()
98 * Matt Dillon : More TCP improvements and RST bug fixes
99 * Matt Dillon : Yet more small nasties remove from the
100 * TCP code (Be very nice to this man if
101 * tcp finally works 100%) 8)
102 * Alan Cox : BSD accept semantics.
103 * Alan Cox : Reset on closedown bug.
104 * Peter De Schrijver : ENOTCONN check missing in tcp_sendto().
105 * Michael Pall : Handle poll() after URG properly in
107 * Michael Pall : Undo the last fix in tcp_read_urg()
108 * (multi URG PUSH broke rlogin).
109 * Michael Pall : Fix the multi URG PUSH problem in
110 * tcp_readable(), poll() after URG
112 * Michael Pall : recv(...,MSG_OOB) never blocks in the
114 * Alan Cox : Changed the semantics of sk->socket to
115 * fix a race and a signal problem with
116 * accept() and async I/O.
117 * Alan Cox : Relaxed the rules on tcp_sendto().
118 * Yury Shevchuk : Really fixed accept() blocking problem.
119 * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for
120 * clients/servers which listen in on
122 * Alan Cox : Cleaned the above up and shrank it to
123 * a sensible code size.
124 * Alan Cox : Self connect lockup fix.
125 * Alan Cox : No connect to multicast.
126 * Ross Biro : Close unaccepted children on master
128 * Alan Cox : Reset tracing code.
129 * Alan Cox : Spurious resets on shutdown.
130 * Alan Cox : Giant 15 minute/60 second timer error
131 * Alan Cox : Small whoops in polling before an
133 * Alan Cox : Kept the state trace facility since
134 * it's handy for debugging.
135 * Alan Cox : More reset handler fixes.
136 * Alan Cox : Started rewriting the code based on
137 * the RFC's for other useful protocol
138 * references see: Comer, KA9Q NOS, and
139 * for a reference on the difference
140 * between specifications and how BSD
141 * works see the 4.4lite source.
142 * A.N.Kuznetsov : Don't time wait on completion of tidy
144 * Linus Torvalds : Fin/Shutdown & copied_seq changes.
145 * Linus Torvalds : Fixed BSD port reuse to work first syn
146 * Alan Cox : Reimplemented timers as per the RFC
147 * and using multiple timers for sanity.
148 * Alan Cox : Small bug fixes, and a lot of new
150 * Alan Cox : Fixed dual reader crash by locking
151 * the buffers (much like datagram.c)
152 * Alan Cox : Fixed stuck sockets in probe. A probe
153 * now gets fed up of retrying without
154 * (even a no space) answer.
155 * Alan Cox : Extracted closing code better
156 * Alan Cox : Fixed the closing state machine to
158 * Alan Cox : More 'per spec' fixes.
159 * Jorge Cwik : Even faster checksumming.
160 * Alan Cox : tcp_data() doesn't ack illegal PSH
161 * only frames. At least one pc tcp stack
163 * Alan Cox : Cache last socket.
164 * Alan Cox : Per route irtt.
165 * Matt Day : poll()->select() match BSD precisely on error
166 * Alan Cox : New buffers
167 * Marc Tamsky : Various sk->prot->retransmits and
168 * sk->retransmits misupdating fixed.
169 * Fixed tcp_write_timeout: stuck close,
170 * and TCP syn retries gets used now.
171 * Mark Yarvis : In tcp_read_wakeup(), don't send an
172 * ack if state is TCP_CLOSED.
173 * Alan Cox : Look up device on a retransmit - routes may
174 * change. Doesn't yet cope with MSS shrink right
176 * Marc Tamsky : Closing in closing fixes.
177 * Mike Shaver : RFC1122 verifications.
178 * Alan Cox : rcv_saddr errors.
179 * Alan Cox : Block double connect().
180 * Alan Cox : Small hooks for enSKIP.
181 * Alexey Kuznetsov: Path MTU discovery.
182 * Alan Cox : Support soft errors.
183 * Alan Cox : Fix MTU discovery pathological case
184 * when the remote claims no mtu!
185 * Marc Tamsky : TCP_CLOSE fix.
186 * Colin (G3TNE) : Send a reset on syn ack replies in
187 * window but wrong (fixes NT lpd problems)
188 * Pedro Roque : Better TCP window handling, delayed ack.
189 * Joerg Reuter : No modification of locked buffers in
190 * tcp_do_retransmit()
191 * Eric Schenk : Changed receiver side silly window
192 * avoidance algorithm to BSD style
193 * algorithm. This doubles throughput
194 * against machines running Solaris,
195 * and seems to result in general
197 * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD
198 * Willy Konynenberg : Transparent proxying support.
199 * Mike McLagan : Routing by source
200 * Keith Owens : Do proper merging with partial SKB's in
201 * tcp_do_sendmsg to avoid burstiness.
202 * Eric Schenk : Fix fast close down bug with
203 * shutdown() followed by close().
204 * Andi Kleen : Make poll agree with SIGIO
205 * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and
206 * lingertime == 0 (RFC 793 ABORT Call)
207 * Hirokazu Takahashi : Use copy_from_user() instead of
208 * csum_and_copy_from_user() if possible.
210 * This program is free software; you can redistribute it and/or
211 * modify it under the terms of the GNU General Public License
212 * as published by the Free Software Foundation; either version
213 * 2 of the License, or(at your option) any later version.
215 * Description of States:
217 * TCP_SYN_SENT sent a connection request, waiting for ack
219 * TCP_SYN_RECV received a connection request, sent ack,
220 * waiting for final ack in three-way handshake.
222 * TCP_ESTABLISHED connection established
224 * TCP_FIN_WAIT1 our side has shutdown, waiting to complete
225 * transmission of remaining buffered data
227 * TCP_FIN_WAIT2 all buffered data sent, waiting for remote
230 * TCP_CLOSING both sides have shutdown but we still have
231 * data we have to finish sending
233 * TCP_TIME_WAIT timeout to catch resent junk before entering
234 * closed, can only be entered from FIN_WAIT2
235 * or CLOSING. Required because the other end
236 * may not have gotten our last ACK causing it
237 * to retransmit the data packet (which we ignore)
239 * TCP_CLOSE_WAIT remote side has shutdown and is waiting for
240 * us to finish writing our data and to shutdown
241 * (we have to close() to move on to LAST_ACK)
243 * TCP_LAST_ACK out side has shutdown after remote has
244 * shutdown. There may still be data in our
245 * buffer that we have to finish sending
247 * TCP_CLOSE socket is finished
250 #include <linux/config.h>
251 #include <linux/module.h>
252 #include <linux/types.h>
253 #include <linux/fcntl.h>
254 #include <linux/poll.h>
255 #include <linux/init.h>
256 #include <linux/smp_lock.h>
257 #include <linux/fs.h>
258 #include <linux/random.h>
259 #include <linux/bootmem.h>
261 #include <net/icmp.h>
263 #include <net/xfrm.h>
267 #include <asm/uaccess.h>
268 #include <asm/ioctls.h>
270 int sysctl_tcp_fin_timeout
= TCP_FIN_TIMEOUT
;
272 DEFINE_SNMP_STAT(struct tcp_mib
, tcp_statistics
);
274 atomic_t tcp_orphan_count
= ATOMIC_INIT(0);
276 int sysctl_tcp_mem
[3];
277 int sysctl_tcp_wmem
[3] = { 4 * 1024, 16 * 1024, 128 * 1024 };
278 int sysctl_tcp_rmem
[3] = { 4 * 1024, 87380, 87380 * 2 };
280 EXPORT_SYMBOL(sysctl_tcp_mem
);
281 EXPORT_SYMBOL(sysctl_tcp_rmem
);
282 EXPORT_SYMBOL(sysctl_tcp_wmem
);
284 atomic_t tcp_memory_allocated
; /* Current allocated memory. */
285 atomic_t tcp_sockets_allocated
; /* Current number of TCP sockets. */
287 EXPORT_SYMBOL(tcp_memory_allocated
);
288 EXPORT_SYMBOL(tcp_sockets_allocated
);
291 * Pressure flag: try to collapse.
292 * Technical note: it is used by multiple contexts non atomically.
293 * All the sk_stream_mem_schedule() is of this nature: accounting
294 * is strict, actions are advisory and have some latency.
296 int tcp_memory_pressure
;
298 EXPORT_SYMBOL(tcp_memory_pressure
);
300 void tcp_enter_memory_pressure(void)
302 if (!tcp_memory_pressure
) {
303 NET_INC_STATS(LINUX_MIB_TCPMEMORYPRESSURES
);
304 tcp_memory_pressure
= 1;
308 EXPORT_SYMBOL(tcp_enter_memory_pressure
);
311 * LISTEN is a special case for poll..
313 static __inline__
unsigned int tcp_listen_poll(struct sock
*sk
,
316 return !reqsk_queue_empty(&inet_csk(sk
)->icsk_accept_queue
) ? (POLLIN
| POLLRDNORM
) : 0;
320 * Wait for a TCP event.
322 * Note that we don't need to lock the socket, as the upper poll layers
323 * take care of normal races (between the test and the event) and we don't
324 * go look at any of the socket buffers directly.
326 unsigned int tcp_poll(struct file
*file
, struct socket
*sock
, poll_table
*wait
)
329 struct sock
*sk
= sock
->sk
;
330 struct tcp_sock
*tp
= tcp_sk(sk
);
332 poll_wait(file
, sk
->sk_sleep
, wait
);
333 if (sk
->sk_state
== TCP_LISTEN
)
334 return tcp_listen_poll(sk
, wait
);
336 /* Socket is not locked. We are protected from async events
337 by poll logic and correct handling of state changes
338 made by another threads is impossible in any case.
346 * POLLHUP is certainly not done right. But poll() doesn't
347 * have a notion of HUP in just one direction, and for a
348 * socket the read side is more interesting.
350 * Some poll() documentation says that POLLHUP is incompatible
351 * with the POLLOUT/POLLWR flags, so somebody should check this
352 * all. But careful, it tends to be safer to return too many
353 * bits than too few, and you can easily break real applications
354 * if you don't tell them that something has hung up!
358 * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and
359 * our fs/select.c). It means that after we received EOF,
360 * poll always returns immediately, making impossible poll() on write()
361 * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP
362 * if and only if shutdown has been made in both directions.
363 * Actually, it is interesting to look how Solaris and DUX
364 * solve this dilemma. I would prefer, if PULLHUP were maskable,
365 * then we could set it on SND_SHUTDOWN. BTW examples given
366 * in Stevens' books assume exactly this behaviour, it explains
367 * why PULLHUP is incompatible with POLLOUT. --ANK
369 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
370 * blocking on fresh not-connected or disconnected socket. --ANK
372 if (sk
->sk_shutdown
== SHUTDOWN_MASK
|| sk
->sk_state
== TCP_CLOSE
)
374 if (sk
->sk_shutdown
& RCV_SHUTDOWN
)
375 mask
|= POLLIN
| POLLRDNORM
;
378 if ((1 << sk
->sk_state
) & ~(TCPF_SYN_SENT
| TCPF_SYN_RECV
)) {
379 /* Potential race condition. If read of tp below will
380 * escape above sk->sk_state, we can be illegally awaken
381 * in SYN_* states. */
382 if ((tp
->rcv_nxt
!= tp
->copied_seq
) &&
383 (tp
->urg_seq
!= tp
->copied_seq
||
384 tp
->rcv_nxt
!= tp
->copied_seq
+ 1 ||
385 sock_flag(sk
, SOCK_URGINLINE
) || !tp
->urg_data
))
386 mask
|= POLLIN
| POLLRDNORM
;
388 if (!(sk
->sk_shutdown
& SEND_SHUTDOWN
)) {
389 if (sk_stream_wspace(sk
) >= sk_stream_min_wspace(sk
)) {
390 mask
|= POLLOUT
| POLLWRNORM
;
391 } else { /* send SIGIO later */
392 set_bit(SOCK_ASYNC_NOSPACE
,
393 &sk
->sk_socket
->flags
);
394 set_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
);
396 /* Race breaker. If space is freed after
397 * wspace test but before the flags are set,
398 * IO signal will be lost.
400 if (sk_stream_wspace(sk
) >= sk_stream_min_wspace(sk
))
401 mask
|= POLLOUT
| POLLWRNORM
;
405 if (tp
->urg_data
& TCP_URG_VALID
)
411 int tcp_ioctl(struct sock
*sk
, int cmd
, unsigned long arg
)
413 struct tcp_sock
*tp
= tcp_sk(sk
);
418 if (sk
->sk_state
== TCP_LISTEN
)
422 if ((1 << sk
->sk_state
) & (TCPF_SYN_SENT
| TCPF_SYN_RECV
))
424 else if (sock_flag(sk
, SOCK_URGINLINE
) ||
426 before(tp
->urg_seq
, tp
->copied_seq
) ||
427 !before(tp
->urg_seq
, tp
->rcv_nxt
)) {
428 answ
= tp
->rcv_nxt
- tp
->copied_seq
;
430 /* Subtract 1, if FIN is in queue. */
431 if (answ
&& !skb_queue_empty(&sk
->sk_receive_queue
))
433 ((struct sk_buff
*)sk
->sk_receive_queue
.prev
)->h
.th
->fin
;
435 answ
= tp
->urg_seq
- tp
->copied_seq
;
439 answ
= tp
->urg_data
&& tp
->urg_seq
== tp
->copied_seq
;
442 if (sk
->sk_state
== TCP_LISTEN
)
445 if ((1 << sk
->sk_state
) & (TCPF_SYN_SENT
| TCPF_SYN_RECV
))
448 answ
= tp
->write_seq
- tp
->snd_una
;
454 return put_user(answ
, (int __user
*)arg
);
458 int tcp_listen_start(struct sock
*sk
)
460 struct inet_sock
*inet
= inet_sk(sk
);
461 struct inet_connection_sock
*icsk
= inet_csk(sk
);
462 int rc
= reqsk_queue_alloc(&icsk
->icsk_accept_queue
, TCP_SYNQ_HSIZE
);
467 sk
->sk_max_ack_backlog
= 0;
468 sk
->sk_ack_backlog
= 0;
469 inet_csk_delack_init(sk
);
471 /* There is race window here: we announce ourselves listening,
472 * but this transition is still not validated by get_port().
473 * It is OK, because this socket enters to hash table only
474 * after validation is complete.
476 sk
->sk_state
= TCP_LISTEN
;
477 if (!sk
->sk_prot
->get_port(sk
, inet
->num
)) {
478 inet
->sport
= htons(inet
->num
);
481 sk
->sk_prot
->hash(sk
);
486 sk
->sk_state
= TCP_CLOSE
;
487 __reqsk_queue_destroy(&icsk
->icsk_accept_queue
);
492 * This routine closes sockets which have been at least partially
493 * opened, but not yet accepted.
496 static void tcp_listen_stop (struct sock
*sk
)
498 struct inet_connection_sock
*icsk
= inet_csk(sk
);
499 struct request_sock
*acc_req
;
500 struct request_sock
*req
;
502 inet_csk_delete_keepalive_timer(sk
);
504 /* make all the listen_opt local to us */
505 acc_req
= reqsk_queue_yank_acceptq(&icsk
->icsk_accept_queue
);
507 /* Following specs, it would be better either to send FIN
508 * (and enter FIN-WAIT-1, it is normal close)
509 * or to send active reset (abort).
510 * Certainly, it is pretty dangerous while synflood, but it is
511 * bad justification for our negligence 8)
512 * To be honest, we are not able to make either
513 * of the variants now. --ANK
515 reqsk_queue_destroy(&icsk
->icsk_accept_queue
);
517 while ((req
= acc_req
) != NULL
) {
518 struct sock
*child
= req
->sk
;
520 acc_req
= req
->dl_next
;
524 BUG_TRAP(!sock_owned_by_user(child
));
527 tcp_disconnect(child
, O_NONBLOCK
);
531 atomic_inc(&tcp_orphan_count
);
533 tcp_destroy_sock(child
);
535 bh_unlock_sock(child
);
539 sk_acceptq_removed(sk
);
542 BUG_TRAP(!sk
->sk_ack_backlog
);
545 static inline void tcp_mark_push(struct tcp_sock
*tp
, struct sk_buff
*skb
)
547 TCP_SKB_CB(skb
)->flags
|= TCPCB_FLAG_PSH
;
548 tp
->pushed_seq
= tp
->write_seq
;
551 static inline int forced_push(struct tcp_sock
*tp
)
553 return after(tp
->write_seq
, tp
->pushed_seq
+ (tp
->max_window
>> 1));
556 static inline void skb_entail(struct sock
*sk
, struct tcp_sock
*tp
,
560 TCP_SKB_CB(skb
)->seq
= tp
->write_seq
;
561 TCP_SKB_CB(skb
)->end_seq
= tp
->write_seq
;
562 TCP_SKB_CB(skb
)->flags
= TCPCB_FLAG_ACK
;
563 TCP_SKB_CB(skb
)->sacked
= 0;
564 skb_header_release(skb
);
565 __skb_queue_tail(&sk
->sk_write_queue
, skb
);
566 sk_charge_skb(sk
, skb
);
567 if (!sk
->sk_send_head
)
568 sk
->sk_send_head
= skb
;
569 if (tp
->nonagle
& TCP_NAGLE_PUSH
)
570 tp
->nonagle
&= ~TCP_NAGLE_PUSH
;
573 static inline void tcp_mark_urg(struct tcp_sock
*tp
, int flags
,
576 if (flags
& MSG_OOB
) {
578 tp
->snd_up
= tp
->write_seq
;
579 TCP_SKB_CB(skb
)->sacked
|= TCPCB_URG
;
583 static inline void tcp_push(struct sock
*sk
, struct tcp_sock
*tp
, int flags
,
584 int mss_now
, int nonagle
)
586 if (sk
->sk_send_head
) {
587 struct sk_buff
*skb
= sk
->sk_write_queue
.prev
;
588 if (!(flags
& MSG_MORE
) || forced_push(tp
))
589 tcp_mark_push(tp
, skb
);
590 tcp_mark_urg(tp
, flags
, skb
);
591 __tcp_push_pending_frames(sk
, tp
, mss_now
,
592 (flags
& MSG_MORE
) ? TCP_NAGLE_CORK
: nonagle
);
596 static ssize_t
do_tcp_sendpages(struct sock
*sk
, struct page
**pages
, int poffset
,
597 size_t psize
, int flags
)
599 struct tcp_sock
*tp
= tcp_sk(sk
);
600 int mss_now
, size_goal
;
603 long timeo
= sock_sndtimeo(sk
, flags
& MSG_DONTWAIT
);
605 /* Wait for a connection to finish. */
606 if ((1 << sk
->sk_state
) & ~(TCPF_ESTABLISHED
| TCPF_CLOSE_WAIT
))
607 if ((err
= sk_stream_wait_connect(sk
, &timeo
)) != 0)
610 clear_bit(SOCK_ASYNC_NOSPACE
, &sk
->sk_socket
->flags
);
612 mss_now
= tcp_current_mss(sk
, !(flags
&MSG_OOB
));
613 size_goal
= tp
->xmit_size_goal
;
617 if (sk
->sk_err
|| (sk
->sk_shutdown
& SEND_SHUTDOWN
))
621 struct sk_buff
*skb
= sk
->sk_write_queue
.prev
;
622 struct page
*page
= pages
[poffset
/ PAGE_SIZE
];
623 int copy
, i
, can_coalesce
;
624 int offset
= poffset
% PAGE_SIZE
;
625 int size
= min_t(size_t, psize
, PAGE_SIZE
- offset
);
627 if (!sk
->sk_send_head
|| (copy
= size_goal
- skb
->len
) <= 0) {
629 if (!sk_stream_memory_free(sk
))
630 goto wait_for_sndbuf
;
632 skb
= sk_stream_alloc_pskb(sk
, 0, 0,
635 goto wait_for_memory
;
637 skb_entail(sk
, tp
, skb
);
644 i
= skb_shinfo(skb
)->nr_frags
;
645 can_coalesce
= skb_can_coalesce(skb
, i
, page
, offset
);
646 if (!can_coalesce
&& i
>= MAX_SKB_FRAGS
) {
647 tcp_mark_push(tp
, skb
);
650 if (sk
->sk_forward_alloc
< copy
&&
651 !sk_stream_mem_schedule(sk
, copy
, 0))
652 goto wait_for_memory
;
655 skb_shinfo(skb
)->frags
[i
- 1].size
+= copy
;
658 skb_fill_page_desc(skb
, i
, page
, offset
, copy
);
662 skb
->data_len
+= copy
;
663 skb
->truesize
+= copy
;
664 sk
->sk_wmem_queued
+= copy
;
665 sk
->sk_forward_alloc
-= copy
;
666 skb
->ip_summed
= CHECKSUM_HW
;
667 tp
->write_seq
+= copy
;
668 TCP_SKB_CB(skb
)->end_seq
+= copy
;
669 skb_shinfo(skb
)->tso_segs
= 0;
672 TCP_SKB_CB(skb
)->flags
&= ~TCPCB_FLAG_PSH
;
676 if (!(psize
-= copy
))
679 if (skb
->len
< mss_now
|| (flags
& MSG_OOB
))
682 if (forced_push(tp
)) {
683 tcp_mark_push(tp
, skb
);
684 __tcp_push_pending_frames(sk
, tp
, mss_now
, TCP_NAGLE_PUSH
);
685 } else if (skb
== sk
->sk_send_head
)
686 tcp_push_one(sk
, mss_now
);
690 set_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
);
693 tcp_push(sk
, tp
, flags
& ~MSG_MORE
, mss_now
, TCP_NAGLE_PUSH
);
695 if ((err
= sk_stream_wait_memory(sk
, &timeo
)) != 0)
698 mss_now
= tcp_current_mss(sk
, !(flags
&MSG_OOB
));
699 size_goal
= tp
->xmit_size_goal
;
704 tcp_push(sk
, tp
, flags
, mss_now
, tp
->nonagle
);
711 return sk_stream_error(sk
, flags
, err
);
714 ssize_t
tcp_sendpage(struct socket
*sock
, struct page
*page
, int offset
,
715 size_t size
, int flags
)
718 struct sock
*sk
= sock
->sk
;
720 #define TCP_ZC_CSUM_FLAGS (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM | NETIF_F_HW_CSUM)
722 if (!(sk
->sk_route_caps
& NETIF_F_SG
) ||
723 !(sk
->sk_route_caps
& TCP_ZC_CSUM_FLAGS
))
724 return sock_no_sendpage(sock
, page
, offset
, size
, flags
);
726 #undef TCP_ZC_CSUM_FLAGS
730 res
= do_tcp_sendpages(sk
, &page
, offset
, size
, flags
);
736 #define TCP_PAGE(sk) (sk->sk_sndmsg_page)
737 #define TCP_OFF(sk) (sk->sk_sndmsg_off)
739 static inline int select_size(struct sock
*sk
, struct tcp_sock
*tp
)
741 int tmp
= tp
->mss_cache
;
743 if (sk
->sk_route_caps
& NETIF_F_SG
) {
744 if (sk
->sk_route_caps
& NETIF_F_TSO
)
747 int pgbreak
= SKB_MAX_HEAD(MAX_TCP_HEADER
);
749 if (tmp
>= pgbreak
&&
750 tmp
<= pgbreak
+ (MAX_SKB_FRAGS
- 1) * PAGE_SIZE
)
758 int tcp_sendmsg(struct kiocb
*iocb
, struct sock
*sk
, struct msghdr
*msg
,
762 struct tcp_sock
*tp
= tcp_sk(sk
);
765 int mss_now
, size_goal
;
772 flags
= msg
->msg_flags
;
773 timeo
= sock_sndtimeo(sk
, flags
& MSG_DONTWAIT
);
775 /* Wait for a connection to finish. */
776 if ((1 << sk
->sk_state
) & ~(TCPF_ESTABLISHED
| TCPF_CLOSE_WAIT
))
777 if ((err
= sk_stream_wait_connect(sk
, &timeo
)) != 0)
780 /* This should be in poll */
781 clear_bit(SOCK_ASYNC_NOSPACE
, &sk
->sk_socket
->flags
);
783 mss_now
= tcp_current_mss(sk
, !(flags
&MSG_OOB
));
784 size_goal
= tp
->xmit_size_goal
;
786 /* Ok commence sending. */
787 iovlen
= msg
->msg_iovlen
;
792 if (sk
->sk_err
|| (sk
->sk_shutdown
& SEND_SHUTDOWN
))
795 while (--iovlen
>= 0) {
796 int seglen
= iov
->iov_len
;
797 unsigned char __user
*from
= iov
->iov_base
;
804 skb
= sk
->sk_write_queue
.prev
;
806 if (!sk
->sk_send_head
||
807 (copy
= size_goal
- skb
->len
) <= 0) {
810 /* Allocate new segment. If the interface is SG,
811 * allocate skb fitting to single page.
813 if (!sk_stream_memory_free(sk
))
814 goto wait_for_sndbuf
;
816 skb
= sk_stream_alloc_pskb(sk
, select_size(sk
, tp
),
817 0, sk
->sk_allocation
);
819 goto wait_for_memory
;
822 * Check whether we can use HW checksum.
824 if (sk
->sk_route_caps
&
825 (NETIF_F_IP_CSUM
| NETIF_F_NO_CSUM
|
827 skb
->ip_summed
= CHECKSUM_HW
;
829 skb_entail(sk
, tp
, skb
);
833 /* Try to append data to the end of skb. */
837 /* Where to copy to? */
838 if (skb_tailroom(skb
) > 0) {
839 /* We have some space in skb head. Superb! */
840 if (copy
> skb_tailroom(skb
))
841 copy
= skb_tailroom(skb
);
842 if ((err
= skb_add_data(skb
, from
, copy
)) != 0)
846 int i
= skb_shinfo(skb
)->nr_frags
;
847 struct page
*page
= TCP_PAGE(sk
);
848 int off
= TCP_OFF(sk
);
850 if (skb_can_coalesce(skb
, i
, page
, off
) &&
852 /* We can extend the last page
855 } else if (i
== MAX_SKB_FRAGS
||
857 !(sk
->sk_route_caps
& NETIF_F_SG
))) {
858 /* Need to add new fragment and cannot
859 * do this because interface is non-SG,
860 * or because all the page slots are
862 tcp_mark_push(tp
, skb
);
865 if (off
== PAGE_SIZE
) {
867 TCP_PAGE(sk
) = page
= NULL
;
872 /* Allocate new cache page. */
873 if (!(page
= sk_stream_alloc_page(sk
)))
874 goto wait_for_memory
;
878 if (copy
> PAGE_SIZE
- off
)
879 copy
= PAGE_SIZE
- off
;
881 /* Time to copy data. We are close to
883 err
= skb_copy_to_page(sk
, from
, skb
, page
,
886 /* If this page was new, give it to the
887 * socket so it does not get leaked.
896 /* Update the skb. */
898 skb_shinfo(skb
)->frags
[i
- 1].size
+=
901 skb_fill_page_desc(skb
, i
, page
, off
, copy
);
904 } else if (off
+ copy
< PAGE_SIZE
) {
910 TCP_OFF(sk
) = off
+ copy
;
914 TCP_SKB_CB(skb
)->flags
&= ~TCPCB_FLAG_PSH
;
916 tp
->write_seq
+= copy
;
917 TCP_SKB_CB(skb
)->end_seq
+= copy
;
918 skb_shinfo(skb
)->tso_segs
= 0;
922 if ((seglen
-= copy
) == 0 && iovlen
== 0)
925 if (skb
->len
< mss_now
|| (flags
& MSG_OOB
))
928 if (forced_push(tp
)) {
929 tcp_mark_push(tp
, skb
);
930 __tcp_push_pending_frames(sk
, tp
, mss_now
, TCP_NAGLE_PUSH
);
931 } else if (skb
== sk
->sk_send_head
)
932 tcp_push_one(sk
, mss_now
);
936 set_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
);
939 tcp_push(sk
, tp
, flags
& ~MSG_MORE
, mss_now
, TCP_NAGLE_PUSH
);
941 if ((err
= sk_stream_wait_memory(sk
, &timeo
)) != 0)
944 mss_now
= tcp_current_mss(sk
, !(flags
&MSG_OOB
));
945 size_goal
= tp
->xmit_size_goal
;
951 tcp_push(sk
, tp
, flags
, mss_now
, tp
->nonagle
);
958 if (sk
->sk_send_head
== skb
)
959 sk
->sk_send_head
= NULL
;
960 __skb_unlink(skb
, &sk
->sk_write_queue
);
961 sk_stream_free_skb(sk
, skb
);
968 err
= sk_stream_error(sk
, flags
, err
);
975 * Handle reading urgent data. BSD has very simple semantics for
976 * this, no blocking and very strange errors 8)
979 static int tcp_recv_urg(struct sock
*sk
, long timeo
,
980 struct msghdr
*msg
, int len
, int flags
,
983 struct tcp_sock
*tp
= tcp_sk(sk
);
985 /* No URG data to read. */
986 if (sock_flag(sk
, SOCK_URGINLINE
) || !tp
->urg_data
||
987 tp
->urg_data
== TCP_URG_READ
)
988 return -EINVAL
; /* Yes this is right ! */
990 if (sk
->sk_state
== TCP_CLOSE
&& !sock_flag(sk
, SOCK_DONE
))
993 if (tp
->urg_data
& TCP_URG_VALID
) {
995 char c
= tp
->urg_data
;
997 if (!(flags
& MSG_PEEK
))
998 tp
->urg_data
= TCP_URG_READ
;
1000 /* Read urgent data. */
1001 msg
->msg_flags
|= MSG_OOB
;
1004 if (!(flags
& MSG_TRUNC
))
1005 err
= memcpy_toiovec(msg
->msg_iov
, &c
, 1);
1008 msg
->msg_flags
|= MSG_TRUNC
;
1010 return err
? -EFAULT
: len
;
1013 if (sk
->sk_state
== TCP_CLOSE
|| (sk
->sk_shutdown
& RCV_SHUTDOWN
))
1016 /* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and
1017 * the available implementations agree in this case:
1018 * this call should never block, independent of the
1019 * blocking state of the socket.
1020 * Mike <pall@rz.uni-karlsruhe.de>
1025 /* Clean up the receive buffer for full frames taken by the user,
1026 * then send an ACK if necessary. COPIED is the number of bytes
1027 * tcp_recvmsg has given to the user so far, it speeds up the
1028 * calculation of whether or not we must ACK for the sake of
1031 static void cleanup_rbuf(struct sock
*sk
, int copied
)
1033 struct tcp_sock
*tp
= tcp_sk(sk
);
1034 int time_to_ack
= 0;
1037 struct sk_buff
*skb
= skb_peek(&sk
->sk_receive_queue
);
1039 BUG_TRAP(!skb
|| before(tp
->copied_seq
, TCP_SKB_CB(skb
)->end_seq
));
1042 if (inet_csk_ack_scheduled(sk
)) {
1043 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
1044 /* Delayed ACKs frequently hit locked sockets during bulk
1046 if (icsk
->icsk_ack
.blocked
||
1047 /* Once-per-two-segments ACK was not sent by tcp_input.c */
1048 tp
->rcv_nxt
- tp
->rcv_wup
> icsk
->icsk_ack
.rcv_mss
||
1050 * If this read emptied read buffer, we send ACK, if
1051 * connection is not bidirectional, user drained
1052 * receive buffer and there was a small segment
1055 (copied
> 0 && (icsk
->icsk_ack
.pending
& ICSK_ACK_PUSHED
) &&
1056 !icsk
->icsk_ack
.pingpong
&& !atomic_read(&sk
->sk_rmem_alloc
)))
1060 /* We send an ACK if we can now advertise a non-zero window
1061 * which has been raised "significantly".
1063 * Even if window raised up to infinity, do not send window open ACK
1064 * in states, where we will not receive more. It is useless.
1066 if (copied
> 0 && !time_to_ack
&& !(sk
->sk_shutdown
& RCV_SHUTDOWN
)) {
1067 __u32 rcv_window_now
= tcp_receive_window(tp
);
1069 /* Optimize, __tcp_select_window() is not cheap. */
1070 if (2*rcv_window_now
<= tp
->window_clamp
) {
1071 __u32 new_window
= __tcp_select_window(sk
);
1073 /* Send ACK now, if this read freed lots of space
1074 * in our buffer. Certainly, new_window is new window.
1075 * We can advertise it now, if it is not less than current one.
1076 * "Lots" means "at least twice" here.
1078 if (new_window
&& new_window
>= 2 * rcv_window_now
)
1086 static void tcp_prequeue_process(struct sock
*sk
)
1088 struct sk_buff
*skb
;
1089 struct tcp_sock
*tp
= tcp_sk(sk
);
1091 NET_INC_STATS_USER(LINUX_MIB_TCPPREQUEUED
);
1093 /* RX process wants to run with disabled BHs, though it is not
1096 while ((skb
= __skb_dequeue(&tp
->ucopy
.prequeue
)) != NULL
)
1097 sk
->sk_backlog_rcv(sk
, skb
);
1100 /* Clear memory counter. */
1101 tp
->ucopy
.memory
= 0;
1104 static inline struct sk_buff
*tcp_recv_skb(struct sock
*sk
, u32 seq
, u32
*off
)
1106 struct sk_buff
*skb
;
1109 skb_queue_walk(&sk
->sk_receive_queue
, skb
) {
1110 offset
= seq
- TCP_SKB_CB(skb
)->seq
;
1113 if (offset
< skb
->len
|| skb
->h
.th
->fin
) {
1122 * This routine provides an alternative to tcp_recvmsg() for routines
1123 * that would like to handle copying from skbuffs directly in 'sendfile'
1126 * - It is assumed that the socket was locked by the caller.
1127 * - The routine does not block.
1128 * - At present, there is no support for reading OOB data
1129 * or for 'peeking' the socket using this routine
1130 * (although both would be easy to implement).
1132 int tcp_read_sock(struct sock
*sk
, read_descriptor_t
*desc
,
1133 sk_read_actor_t recv_actor
)
1135 struct sk_buff
*skb
;
1136 struct tcp_sock
*tp
= tcp_sk(sk
);
1137 u32 seq
= tp
->copied_seq
;
1141 if (sk
->sk_state
== TCP_LISTEN
)
1143 while ((skb
= tcp_recv_skb(sk
, seq
, &offset
)) != NULL
) {
1144 if (offset
< skb
->len
) {
1147 len
= skb
->len
- offset
;
1148 /* Stop reading if we hit a patch of urgent data */
1150 u32 urg_offset
= tp
->urg_seq
- seq
;
1151 if (urg_offset
< len
)
1156 used
= recv_actor(desc
, skb
, offset
, len
);
1162 if (offset
!= skb
->len
)
1165 if (skb
->h
.th
->fin
) {
1166 sk_eat_skb(sk
, skb
);
1170 sk_eat_skb(sk
, skb
);
1174 tp
->copied_seq
= seq
;
1176 tcp_rcv_space_adjust(sk
);
1178 /* Clean up data we have read: This will do ACK frames. */
1180 cleanup_rbuf(sk
, copied
);
1185 * This routine copies from a sock struct into the user buffer.
1187 * Technical note: in 2.3 we work on _locked_ socket, so that
1188 * tricks with *seq access order and skb->users are not required.
1189 * Probably, code can be easily improved even more.
1192 int tcp_recvmsg(struct kiocb
*iocb
, struct sock
*sk
, struct msghdr
*msg
,
1193 size_t len
, int nonblock
, int flags
, int *addr_len
)
1195 struct tcp_sock
*tp
= tcp_sk(sk
);
1201 int target
; /* Read at least this many bytes */
1203 struct task_struct
*user_recv
= NULL
;
1207 TCP_CHECK_TIMER(sk
);
1210 if (sk
->sk_state
== TCP_LISTEN
)
1213 timeo
= sock_rcvtimeo(sk
, nonblock
);
1215 /* Urgent data needs to be handled specially. */
1216 if (flags
& MSG_OOB
)
1219 seq
= &tp
->copied_seq
;
1220 if (flags
& MSG_PEEK
) {
1221 peek_seq
= tp
->copied_seq
;
1225 target
= sock_rcvlowat(sk
, flags
& MSG_WAITALL
, len
);
1228 struct sk_buff
*skb
;
1231 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
1232 if (tp
->urg_data
&& tp
->urg_seq
== *seq
) {
1235 if (signal_pending(current
)) {
1236 copied
= timeo
? sock_intr_errno(timeo
) : -EAGAIN
;
1241 /* Next get a buffer. */
1243 skb
= skb_peek(&sk
->sk_receive_queue
);
1248 /* Now that we have two receive queues this
1251 if (before(*seq
, TCP_SKB_CB(skb
)->seq
)) {
1252 printk(KERN_INFO
"recvmsg bug: copied %X "
1253 "seq %X\n", *seq
, TCP_SKB_CB(skb
)->seq
);
1256 offset
= *seq
- TCP_SKB_CB(skb
)->seq
;
1259 if (offset
< skb
->len
)
1263 BUG_TRAP(flags
& MSG_PEEK
);
1265 } while (skb
!= (struct sk_buff
*)&sk
->sk_receive_queue
);
1267 /* Well, if we have backlog, try to process it now yet. */
1269 if (copied
>= target
&& !sk
->sk_backlog
.tail
)
1274 sk
->sk_state
== TCP_CLOSE
||
1275 (sk
->sk_shutdown
& RCV_SHUTDOWN
) ||
1277 signal_pending(current
) ||
1281 if (sock_flag(sk
, SOCK_DONE
))
1285 copied
= sock_error(sk
);
1289 if (sk
->sk_shutdown
& RCV_SHUTDOWN
)
1292 if (sk
->sk_state
== TCP_CLOSE
) {
1293 if (!sock_flag(sk
, SOCK_DONE
)) {
1294 /* This occurs when user tries to read
1295 * from never connected socket.
1308 if (signal_pending(current
)) {
1309 copied
= sock_intr_errno(timeo
);
1314 cleanup_rbuf(sk
, copied
);
1316 if (!sysctl_tcp_low_latency
&& tp
->ucopy
.task
== user_recv
) {
1317 /* Install new reader */
1318 if (!user_recv
&& !(flags
& (MSG_TRUNC
| MSG_PEEK
))) {
1319 user_recv
= current
;
1320 tp
->ucopy
.task
= user_recv
;
1321 tp
->ucopy
.iov
= msg
->msg_iov
;
1324 tp
->ucopy
.len
= len
;
1326 BUG_TRAP(tp
->copied_seq
== tp
->rcv_nxt
||
1327 (flags
& (MSG_PEEK
| MSG_TRUNC
)));
1329 /* Ugly... If prequeue is not empty, we have to
1330 * process it before releasing socket, otherwise
1331 * order will be broken at second iteration.
1332 * More elegant solution is required!!!
1334 * Look: we have the following (pseudo)queues:
1336 * 1. packets in flight
1341 * Each queue can be processed only if the next ones
1342 * are empty. At this point we have empty receive_queue.
1343 * But prequeue _can_ be not empty after 2nd iteration,
1344 * when we jumped to start of loop because backlog
1345 * processing added something to receive_queue.
1346 * We cannot release_sock(), because backlog contains
1347 * packets arrived _after_ prequeued ones.
1349 * Shortly, algorithm is clear --- to process all
1350 * the queues in order. We could make it more directly,
1351 * requeueing packets from backlog to prequeue, if
1352 * is not empty. It is more elegant, but eats cycles,
1355 if (!skb_queue_empty(&tp
->ucopy
.prequeue
))
1358 /* __ Set realtime policy in scheduler __ */
1361 if (copied
>= target
) {
1362 /* Do not sleep, just process backlog. */
1366 sk_wait_data(sk
, &timeo
);
1371 /* __ Restore normal policy in scheduler __ */
1373 if ((chunk
= len
- tp
->ucopy
.len
) != 0) {
1374 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG
, chunk
);
1379 if (tp
->rcv_nxt
== tp
->copied_seq
&&
1380 !skb_queue_empty(&tp
->ucopy
.prequeue
)) {
1382 tcp_prequeue_process(sk
);
1384 if ((chunk
= len
- tp
->ucopy
.len
) != 0) {
1385 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE
, chunk
);
1391 if ((flags
& MSG_PEEK
) && peek_seq
!= tp
->copied_seq
) {
1392 if (net_ratelimit())
1393 printk(KERN_DEBUG
"TCP(%s:%d): Application bug, race in MSG_PEEK.\n",
1394 current
->comm
, current
->pid
);
1395 peek_seq
= tp
->copied_seq
;
1400 /* Ok so how much can we use? */
1401 used
= skb
->len
- offset
;
1405 /* Do we have urgent data here? */
1407 u32 urg_offset
= tp
->urg_seq
- *seq
;
1408 if (urg_offset
< used
) {
1410 if (!sock_flag(sk
, SOCK_URGINLINE
)) {
1422 if (!(flags
& MSG_TRUNC
)) {
1423 err
= skb_copy_datagram_iovec(skb
, offset
,
1424 msg
->msg_iov
, used
);
1426 /* Exception. Bailout! */
1437 tcp_rcv_space_adjust(sk
);
1440 if (tp
->urg_data
&& after(tp
->copied_seq
, tp
->urg_seq
)) {
1442 tcp_fast_path_check(sk
, tp
);
1444 if (used
+ offset
< skb
->len
)
1449 if (!(flags
& MSG_PEEK
))
1450 sk_eat_skb(sk
, skb
);
1454 /* Process the FIN. */
1456 if (!(flags
& MSG_PEEK
))
1457 sk_eat_skb(sk
, skb
);
1462 if (!skb_queue_empty(&tp
->ucopy
.prequeue
)) {
1465 tp
->ucopy
.len
= copied
> 0 ? len
: 0;
1467 tcp_prequeue_process(sk
);
1469 if (copied
> 0 && (chunk
= len
- tp
->ucopy
.len
) != 0) {
1470 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE
, chunk
);
1476 tp
->ucopy
.task
= NULL
;
1480 /* According to UNIX98, msg_name/msg_namelen are ignored
1481 * on connected socket. I was just happy when found this 8) --ANK
1484 /* Clean up data we have read: This will do ACK frames. */
1485 cleanup_rbuf(sk
, copied
);
1487 TCP_CHECK_TIMER(sk
);
1492 TCP_CHECK_TIMER(sk
);
1497 err
= tcp_recv_urg(sk
, timeo
, msg
, len
, flags
, addr_len
);
1502 * State processing on a close. This implements the state shift for
1503 * sending our FIN frame. Note that we only send a FIN for some
1504 * states. A shutdown() may have already sent the FIN, or we may be
1508 static unsigned char new_state
[16] = {
1509 /* current state: new state: action: */
1510 /* (Invalid) */ TCP_CLOSE
,
1511 /* TCP_ESTABLISHED */ TCP_FIN_WAIT1
| TCP_ACTION_FIN
,
1512 /* TCP_SYN_SENT */ TCP_CLOSE
,
1513 /* TCP_SYN_RECV */ TCP_FIN_WAIT1
| TCP_ACTION_FIN
,
1514 /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1
,
1515 /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2
,
1516 /* TCP_TIME_WAIT */ TCP_CLOSE
,
1517 /* TCP_CLOSE */ TCP_CLOSE
,
1518 /* TCP_CLOSE_WAIT */ TCP_LAST_ACK
| TCP_ACTION_FIN
,
1519 /* TCP_LAST_ACK */ TCP_LAST_ACK
,
1520 /* TCP_LISTEN */ TCP_CLOSE
,
1521 /* TCP_CLOSING */ TCP_CLOSING
,
1524 static int tcp_close_state(struct sock
*sk
)
1526 int next
= (int)new_state
[sk
->sk_state
];
1527 int ns
= next
& TCP_STATE_MASK
;
1529 tcp_set_state(sk
, ns
);
1531 return next
& TCP_ACTION_FIN
;
1535 * Shutdown the sending side of a connection. Much like close except
1536 * that we don't receive shut down or set_sock_flag(sk, SOCK_DEAD).
1539 void tcp_shutdown(struct sock
*sk
, int how
)
1541 /* We need to grab some memory, and put together a FIN,
1542 * and then put it into the queue to be sent.
1543 * Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
1545 if (!(how
& SEND_SHUTDOWN
))
1548 /* If we've already sent a FIN, or it's a closed state, skip this. */
1549 if ((1 << sk
->sk_state
) &
1550 (TCPF_ESTABLISHED
| TCPF_SYN_SENT
|
1551 TCPF_SYN_RECV
| TCPF_CLOSE_WAIT
)) {
1552 /* Clear out any half completed packets. FIN if needed. */
1553 if (tcp_close_state(sk
))
1559 * At this point, there should be no process reference to this
1560 * socket, and thus no user references at all. Therefore we
1561 * can assume the socket waitqueue is inactive and nobody will
1562 * try to jump onto it.
1564 void tcp_destroy_sock(struct sock
*sk
)
1566 BUG_TRAP(sk
->sk_state
== TCP_CLOSE
);
1567 BUG_TRAP(sock_flag(sk
, SOCK_DEAD
));
1569 /* It cannot be in hash table! */
1570 BUG_TRAP(sk_unhashed(sk
));
1572 /* If it has not 0 inet_sk(sk)->num, it must be bound */
1573 BUG_TRAP(!inet_sk(sk
)->num
|| inet_csk(sk
)->icsk_bind_hash
);
1575 sk
->sk_prot
->destroy(sk
);
1577 sk_stream_kill_queues(sk
);
1579 xfrm_sk_free_policy(sk
);
1581 sk_refcnt_debug_release(sk
);
1583 atomic_dec(&tcp_orphan_count
);
1587 void tcp_close(struct sock
*sk
, long timeout
)
1589 struct sk_buff
*skb
;
1590 int data_was_unread
= 0;
1593 sk
->sk_shutdown
= SHUTDOWN_MASK
;
1595 if (sk
->sk_state
== TCP_LISTEN
) {
1596 tcp_set_state(sk
, TCP_CLOSE
);
1599 tcp_listen_stop(sk
);
1601 goto adjudge_to_death
;
1604 /* We need to flush the recv. buffs. We do this only on the
1605 * descriptor close, not protocol-sourced closes, because the
1606 * reader process may not have drained the data yet!
1608 while ((skb
= __skb_dequeue(&sk
->sk_receive_queue
)) != NULL
) {
1609 u32 len
= TCP_SKB_CB(skb
)->end_seq
- TCP_SKB_CB(skb
)->seq
-
1611 data_was_unread
+= len
;
1615 sk_stream_mem_reclaim(sk
);
1617 /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section
1618 * 3.10, we send a RST here because data was lost. To
1619 * witness the awful effects of the old behavior of always
1620 * doing a FIN, run an older 2.1.x kernel or 2.0.x, start
1621 * a bulk GET in an FTP client, suspend the process, wait
1622 * for the client to advertise a zero window, then kill -9
1623 * the FTP client, wheee... Note: timeout is always zero
1626 if (data_was_unread
) {
1627 /* Unread data was tossed, zap the connection. */
1628 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONCLOSE
);
1629 tcp_set_state(sk
, TCP_CLOSE
);
1630 tcp_send_active_reset(sk
, GFP_KERNEL
);
1631 } else if (sock_flag(sk
, SOCK_LINGER
) && !sk
->sk_lingertime
) {
1632 /* Check zero linger _after_ checking for unread data. */
1633 sk
->sk_prot
->disconnect(sk
, 0);
1634 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONDATA
);
1635 } else if (tcp_close_state(sk
)) {
1636 /* We FIN if the application ate all the data before
1637 * zapping the connection.
1640 /* RED-PEN. Formally speaking, we have broken TCP state
1641 * machine. State transitions:
1643 * TCP_ESTABLISHED -> TCP_FIN_WAIT1
1644 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible)
1645 * TCP_CLOSE_WAIT -> TCP_LAST_ACK
1647 * are legal only when FIN has been sent (i.e. in window),
1648 * rather than queued out of window. Purists blame.
1650 * F.e. "RFC state" is ESTABLISHED,
1651 * if Linux state is FIN-WAIT-1, but FIN is still not sent.
1653 * The visible declinations are that sometimes
1654 * we enter time-wait state, when it is not required really
1655 * (harmless), do not send active resets, when they are
1656 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
1657 * they look as CLOSING or LAST_ACK for Linux)
1658 * Probably, I missed some more holelets.
1664 sk_stream_wait_close(sk
, timeout
);
1667 /* It is the last release_sock in its life. It will remove backlog. */
1671 /* Now socket is owned by kernel and we acquire BH lock
1672 to finish close. No need to check for user refs.
1676 BUG_TRAP(!sock_owned_by_user(sk
));
1681 /* This is a (useful) BSD violating of the RFC. There is a
1682 * problem with TCP as specified in that the other end could
1683 * keep a socket open forever with no application left this end.
1684 * We use a 3 minute timeout (about the same as BSD) then kill
1685 * our end. If they send after that then tough - BUT: long enough
1686 * that we won't make the old 4*rto = almost no time - whoops
1689 * Nope, it was not mistake. It is really desired behaviour
1690 * f.e. on http servers, when such sockets are useless, but
1691 * consume significant resources. Let's do it with special
1692 * linger2 option. --ANK
1695 if (sk
->sk_state
== TCP_FIN_WAIT2
) {
1696 struct tcp_sock
*tp
= tcp_sk(sk
);
1697 if (tp
->linger2
< 0) {
1698 tcp_set_state(sk
, TCP_CLOSE
);
1699 tcp_send_active_reset(sk
, GFP_ATOMIC
);
1700 NET_INC_STATS_BH(LINUX_MIB_TCPABORTONLINGER
);
1702 const int tmo
= tcp_fin_time(sk
);
1704 if (tmo
> TCP_TIMEWAIT_LEN
) {
1705 inet_csk_reset_keepalive_timer(sk
, tcp_fin_time(sk
));
1707 atomic_inc(&tcp_orphan_count
);
1708 tcp_time_wait(sk
, TCP_FIN_WAIT2
, tmo
);
1713 if (sk
->sk_state
!= TCP_CLOSE
) {
1714 sk_stream_mem_reclaim(sk
);
1715 if (atomic_read(&tcp_orphan_count
) > sysctl_tcp_max_orphans
||
1716 (sk
->sk_wmem_queued
> SOCK_MIN_SNDBUF
&&
1717 atomic_read(&tcp_memory_allocated
) > sysctl_tcp_mem
[2])) {
1718 if (net_ratelimit())
1719 printk(KERN_INFO
"TCP: too many of orphaned "
1721 tcp_set_state(sk
, TCP_CLOSE
);
1722 tcp_send_active_reset(sk
, GFP_ATOMIC
);
1723 NET_INC_STATS_BH(LINUX_MIB_TCPABORTONMEMORY
);
1726 atomic_inc(&tcp_orphan_count
);
1728 if (sk
->sk_state
== TCP_CLOSE
)
1729 tcp_destroy_sock(sk
);
1730 /* Otherwise, socket is reprieved until protocol close. */
1738 /* These states need RST on ABORT according to RFC793 */
1740 static inline int tcp_need_reset(int state
)
1742 return (1 << state
) &
1743 (TCPF_ESTABLISHED
| TCPF_CLOSE_WAIT
| TCPF_FIN_WAIT1
|
1744 TCPF_FIN_WAIT2
| TCPF_SYN_RECV
);
1747 int tcp_disconnect(struct sock
*sk
, int flags
)
1749 struct inet_sock
*inet
= inet_sk(sk
);
1750 struct inet_connection_sock
*icsk
= inet_csk(sk
);
1751 struct tcp_sock
*tp
= tcp_sk(sk
);
1753 int old_state
= sk
->sk_state
;
1755 if (old_state
!= TCP_CLOSE
)
1756 tcp_set_state(sk
, TCP_CLOSE
);
1758 /* ABORT function of RFC793 */
1759 if (old_state
== TCP_LISTEN
) {
1760 tcp_listen_stop(sk
);
1761 } else if (tcp_need_reset(old_state
) ||
1762 (tp
->snd_nxt
!= tp
->write_seq
&&
1763 (1 << old_state
) & (TCPF_CLOSING
| TCPF_LAST_ACK
))) {
1764 /* The last check adjusts for discrepance of Linux wrt. RFC
1767 tcp_send_active_reset(sk
, gfp_any());
1768 sk
->sk_err
= ECONNRESET
;
1769 } else if (old_state
== TCP_SYN_SENT
)
1770 sk
->sk_err
= ECONNRESET
;
1772 tcp_clear_xmit_timers(sk
);
1773 __skb_queue_purge(&sk
->sk_receive_queue
);
1774 sk_stream_writequeue_purge(sk
);
1775 __skb_queue_purge(&tp
->out_of_order_queue
);
1779 if (!(sk
->sk_userlocks
& SOCK_BINDADDR_LOCK
))
1780 inet_reset_saddr(sk
);
1782 sk
->sk_shutdown
= 0;
1783 sock_reset_flag(sk
, SOCK_DONE
);
1785 if ((tp
->write_seq
+= tp
->max_window
+ 2) == 0)
1787 icsk
->icsk_backoff
= 0;
1790 tp
->packets_out
= 0;
1791 tp
->snd_ssthresh
= 0x7fffffff;
1792 tp
->snd_cwnd_cnt
= 0;
1793 tcp_set_ca_state(tp
, TCP_CA_Open
);
1794 tcp_clear_retrans(tp
);
1795 inet_csk_delack_init(sk
);
1796 sk
->sk_send_head
= NULL
;
1797 tp
->rx_opt
.saw_tstamp
= 0;
1798 tcp_sack_reset(&tp
->rx_opt
);
1801 BUG_TRAP(!inet
->num
|| icsk
->icsk_bind_hash
);
1803 sk
->sk_error_report(sk
);
1808 * Wait for an incoming connection, avoid race
1809 * conditions. This must be called with the socket locked.
1811 static int wait_for_connect(struct sock
*sk
, long timeo
)
1813 struct inet_connection_sock
*icsk
= inet_csk(sk
);
1818 * True wake-one mechanism for incoming connections: only
1819 * one process gets woken up, not the 'whole herd'.
1820 * Since we do not 'race & poll' for established sockets
1821 * anymore, the common case will execute the loop only once.
1823 * Subtle issue: "add_wait_queue_exclusive()" will be added
1824 * after any current non-exclusive waiters, and we know that
1825 * it will always _stay_ after any new non-exclusive waiters
1826 * because all non-exclusive waiters are added at the
1827 * beginning of the wait-queue. As such, it's ok to "drop"
1828 * our exclusiveness temporarily when we get woken up without
1829 * having to remove and re-insert us on the wait queue.
1832 prepare_to_wait_exclusive(sk
->sk_sleep
, &wait
,
1833 TASK_INTERRUPTIBLE
);
1835 if (reqsk_queue_empty(&icsk
->icsk_accept_queue
))
1836 timeo
= schedule_timeout(timeo
);
1839 if (!reqsk_queue_empty(&icsk
->icsk_accept_queue
))
1842 if (sk
->sk_state
!= TCP_LISTEN
)
1844 err
= sock_intr_errno(timeo
);
1845 if (signal_pending(current
))
1851 finish_wait(sk
->sk_sleep
, &wait
);
1856 * This will accept the next outstanding connection.
1859 struct sock
*inet_csk_accept(struct sock
*sk
, int flags
, int *err
)
1861 struct inet_connection_sock
*icsk
= inet_csk(sk
);
1867 /* We need to make sure that this socket is listening,
1868 * and that it has something pending.
1871 if (sk
->sk_state
!= TCP_LISTEN
)
1874 /* Find already established connection */
1875 if (reqsk_queue_empty(&icsk
->icsk_accept_queue
)) {
1876 long timeo
= sock_rcvtimeo(sk
, flags
& O_NONBLOCK
);
1878 /* If this is a non blocking socket don't sleep */
1883 error
= wait_for_connect(sk
, timeo
);
1888 newsk
= reqsk_queue_get_child(&icsk
->icsk_accept_queue
, sk
);
1889 BUG_TRAP(newsk
->sk_state
!= TCP_SYN_RECV
);
1900 * Socket option code for TCP.
1902 int tcp_setsockopt(struct sock
*sk
, int level
, int optname
, char __user
*optval
,
1905 struct tcp_sock
*tp
= tcp_sk(sk
);
1906 struct inet_connection_sock
*icsk
= inet_csk(sk
);
1910 if (level
!= SOL_TCP
)
1911 return tp
->af_specific
->setsockopt(sk
, level
, optname
,
1914 /* This is a string value all the others are int's */
1915 if (optname
== TCP_CONGESTION
) {
1916 char name
[TCP_CA_NAME_MAX
];
1921 val
= strncpy_from_user(name
, optval
,
1922 min(TCP_CA_NAME_MAX
-1, optlen
));
1928 err
= tcp_set_congestion_control(tp
, name
);
1933 if (optlen
< sizeof(int))
1936 if (get_user(val
, (int __user
*)optval
))
1943 /* Values greater than interface MTU won't take effect. However
1944 * at the point when this call is done we typically don't yet
1945 * know which interface is going to be used */
1946 if (val
< 8 || val
> MAX_TCP_WINDOW
) {
1950 tp
->rx_opt
.user_mss
= val
;
1955 /* TCP_NODELAY is weaker than TCP_CORK, so that
1956 * this option on corked socket is remembered, but
1957 * it is not activated until cork is cleared.
1959 * However, when TCP_NODELAY is set we make
1960 * an explicit push, which overrides even TCP_CORK
1961 * for currently queued segments.
1963 tp
->nonagle
|= TCP_NAGLE_OFF
|TCP_NAGLE_PUSH
;
1964 tcp_push_pending_frames(sk
, tp
);
1966 tp
->nonagle
&= ~TCP_NAGLE_OFF
;
1971 /* When set indicates to always queue non-full frames.
1972 * Later the user clears this option and we transmit
1973 * any pending partial frames in the queue. This is
1974 * meant to be used alongside sendfile() to get properly
1975 * filled frames when the user (for example) must write
1976 * out headers with a write() call first and then use
1977 * sendfile to send out the data parts.
1979 * TCP_CORK can be set together with TCP_NODELAY and it is
1980 * stronger than TCP_NODELAY.
1983 tp
->nonagle
|= TCP_NAGLE_CORK
;
1985 tp
->nonagle
&= ~TCP_NAGLE_CORK
;
1986 if (tp
->nonagle
&TCP_NAGLE_OFF
)
1987 tp
->nonagle
|= TCP_NAGLE_PUSH
;
1988 tcp_push_pending_frames(sk
, tp
);
1993 if (val
< 1 || val
> MAX_TCP_KEEPIDLE
)
1996 tp
->keepalive_time
= val
* HZ
;
1997 if (sock_flag(sk
, SOCK_KEEPOPEN
) &&
1998 !((1 << sk
->sk_state
) &
1999 (TCPF_CLOSE
| TCPF_LISTEN
))) {
2000 __u32 elapsed
= tcp_time_stamp
- tp
->rcv_tstamp
;
2001 if (tp
->keepalive_time
> elapsed
)
2002 elapsed
= tp
->keepalive_time
- elapsed
;
2005 inet_csk_reset_keepalive_timer(sk
, elapsed
);
2010 if (val
< 1 || val
> MAX_TCP_KEEPINTVL
)
2013 tp
->keepalive_intvl
= val
* HZ
;
2016 if (val
< 1 || val
> MAX_TCP_KEEPCNT
)
2019 tp
->keepalive_probes
= val
;
2022 if (val
< 1 || val
> MAX_TCP_SYNCNT
)
2025 icsk
->icsk_syn_retries
= val
;
2031 else if (val
> sysctl_tcp_fin_timeout
/ HZ
)
2034 tp
->linger2
= val
* HZ
;
2037 case TCP_DEFER_ACCEPT
:
2038 tp
->defer_accept
= 0;
2040 /* Translate value in seconds to number of
2042 while (tp
->defer_accept
< 32 &&
2043 val
> ((TCP_TIMEOUT_INIT
/ HZ
) <<
2050 case TCP_WINDOW_CLAMP
:
2052 if (sk
->sk_state
!= TCP_CLOSE
) {
2056 tp
->window_clamp
= 0;
2058 tp
->window_clamp
= val
< SOCK_MIN_RCVBUF
/ 2 ?
2059 SOCK_MIN_RCVBUF
/ 2 : val
;
2064 icsk
->icsk_ack
.pingpong
= 1;
2066 icsk
->icsk_ack
.pingpong
= 0;
2067 if ((1 << sk
->sk_state
) &
2068 (TCPF_ESTABLISHED
| TCPF_CLOSE_WAIT
) &&
2069 inet_csk_ack_scheduled(sk
)) {
2070 icsk
->icsk_ack
.pending
|= ICSK_ACK_PUSHED
;
2071 cleanup_rbuf(sk
, 1);
2073 icsk
->icsk_ack
.pingpong
= 1;
2086 /* Return information about state of tcp endpoint in API format. */
2087 void tcp_get_info(struct sock
*sk
, struct tcp_info
*info
)
2089 struct tcp_sock
*tp
= tcp_sk(sk
);
2090 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
2091 u32 now
= tcp_time_stamp
;
2093 memset(info
, 0, sizeof(*info
));
2095 info
->tcpi_state
= sk
->sk_state
;
2096 info
->tcpi_ca_state
= tp
->ca_state
;
2097 info
->tcpi_retransmits
= icsk
->icsk_retransmits
;
2098 info
->tcpi_probes
= tp
->probes_out
;
2099 info
->tcpi_backoff
= icsk
->icsk_backoff
;
2101 if (tp
->rx_opt
.tstamp_ok
)
2102 info
->tcpi_options
|= TCPI_OPT_TIMESTAMPS
;
2103 if (tp
->rx_opt
.sack_ok
)
2104 info
->tcpi_options
|= TCPI_OPT_SACK
;
2105 if (tp
->rx_opt
.wscale_ok
) {
2106 info
->tcpi_options
|= TCPI_OPT_WSCALE
;
2107 info
->tcpi_snd_wscale
= tp
->rx_opt
.snd_wscale
;
2108 info
->tcpi_rcv_wscale
= tp
->rx_opt
.rcv_wscale
;
2111 if (tp
->ecn_flags
&TCP_ECN_OK
)
2112 info
->tcpi_options
|= TCPI_OPT_ECN
;
2114 info
->tcpi_rto
= jiffies_to_usecs(icsk
->icsk_rto
);
2115 info
->tcpi_ato
= jiffies_to_usecs(icsk
->icsk_ack
.ato
);
2116 info
->tcpi_snd_mss
= tp
->mss_cache
;
2117 info
->tcpi_rcv_mss
= icsk
->icsk_ack
.rcv_mss
;
2119 info
->tcpi_unacked
= tp
->packets_out
;
2120 info
->tcpi_sacked
= tp
->sacked_out
;
2121 info
->tcpi_lost
= tp
->lost_out
;
2122 info
->tcpi_retrans
= tp
->retrans_out
;
2123 info
->tcpi_fackets
= tp
->fackets_out
;
2125 info
->tcpi_last_data_sent
= jiffies_to_msecs(now
- tp
->lsndtime
);
2126 info
->tcpi_last_data_recv
= jiffies_to_msecs(now
- icsk
->icsk_ack
.lrcvtime
);
2127 info
->tcpi_last_ack_recv
= jiffies_to_msecs(now
- tp
->rcv_tstamp
);
2129 info
->tcpi_pmtu
= tp
->pmtu_cookie
;
2130 info
->tcpi_rcv_ssthresh
= tp
->rcv_ssthresh
;
2131 info
->tcpi_rtt
= jiffies_to_usecs(tp
->srtt
)>>3;
2132 info
->tcpi_rttvar
= jiffies_to_usecs(tp
->mdev
)>>2;
2133 info
->tcpi_snd_ssthresh
= tp
->snd_ssthresh
;
2134 info
->tcpi_snd_cwnd
= tp
->snd_cwnd
;
2135 info
->tcpi_advmss
= tp
->advmss
;
2136 info
->tcpi_reordering
= tp
->reordering
;
2138 info
->tcpi_rcv_rtt
= jiffies_to_usecs(tp
->rcv_rtt_est
.rtt
)>>3;
2139 info
->tcpi_rcv_space
= tp
->rcvq_space
.space
;
2141 info
->tcpi_total_retrans
= tp
->total_retrans
;
2144 EXPORT_SYMBOL_GPL(tcp_get_info
);
2146 int tcp_getsockopt(struct sock
*sk
, int level
, int optname
, char __user
*optval
,
2149 struct tcp_sock
*tp
= tcp_sk(sk
);
2152 if (level
!= SOL_TCP
)
2153 return tp
->af_specific
->getsockopt(sk
, level
, optname
,
2156 if (get_user(len
, optlen
))
2159 len
= min_t(unsigned int, len
, sizeof(int));
2166 val
= tp
->mss_cache
;
2167 if (!val
&& ((1 << sk
->sk_state
) & (TCPF_CLOSE
| TCPF_LISTEN
)))
2168 val
= tp
->rx_opt
.user_mss
;
2171 val
= !!(tp
->nonagle
&TCP_NAGLE_OFF
);
2174 val
= !!(tp
->nonagle
&TCP_NAGLE_CORK
);
2177 val
= (tp
->keepalive_time
? : sysctl_tcp_keepalive_time
) / HZ
;
2180 val
= (tp
->keepalive_intvl
? : sysctl_tcp_keepalive_intvl
) / HZ
;
2183 val
= tp
->keepalive_probes
? : sysctl_tcp_keepalive_probes
;
2186 val
= inet_csk(sk
)->icsk_syn_retries
? : sysctl_tcp_syn_retries
;
2191 val
= (val
? : sysctl_tcp_fin_timeout
) / HZ
;
2193 case TCP_DEFER_ACCEPT
:
2194 val
= !tp
->defer_accept
? 0 : ((TCP_TIMEOUT_INIT
/ HZ
) <<
2195 (tp
->defer_accept
- 1));
2197 case TCP_WINDOW_CLAMP
:
2198 val
= tp
->window_clamp
;
2201 struct tcp_info info
;
2203 if (get_user(len
, optlen
))
2206 tcp_get_info(sk
, &info
);
2208 len
= min_t(unsigned int, len
, sizeof(info
));
2209 if (put_user(len
, optlen
))
2211 if (copy_to_user(optval
, &info
, len
))
2216 val
= !inet_csk(sk
)->icsk_ack
.pingpong
;
2219 case TCP_CONGESTION
:
2220 if (get_user(len
, optlen
))
2222 len
= min_t(unsigned int, len
, TCP_CA_NAME_MAX
);
2223 if (put_user(len
, optlen
))
2225 if (copy_to_user(optval
, tp
->ca_ops
->name
, len
))
2229 return -ENOPROTOOPT
;
2232 if (put_user(len
, optlen
))
2234 if (copy_to_user(optval
, &val
, len
))
2240 extern void __skb_cb_too_small_for_tcp(int, int);
2241 extern struct tcp_congestion_ops tcp_reno
;
2243 static __initdata
unsigned long thash_entries
;
2244 static int __init
set_thash_entries(char *str
)
2248 thash_entries
= simple_strtoul(str
, &str
, 0);
2251 __setup("thash_entries=", set_thash_entries
);
2253 void __init
tcp_init(void)
2255 struct sk_buff
*skb
= NULL
;
2258 if (sizeof(struct tcp_skb_cb
) > sizeof(skb
->cb
))
2259 __skb_cb_too_small_for_tcp(sizeof(struct tcp_skb_cb
),
2262 tcp_hashinfo
.bind_bucket_cachep
=
2263 kmem_cache_create("tcp_bind_bucket",
2264 sizeof(struct inet_bind_bucket
), 0,
2265 SLAB_HWCACHE_ALIGN
, NULL
, NULL
);
2266 if (!tcp_hashinfo
.bind_bucket_cachep
)
2267 panic("tcp_init: Cannot alloc tcp_bind_bucket cache.");
2269 /* Size and allocate the main established and bind bucket
2272 * The methodology is similar to that of the buffer cache.
2274 tcp_hashinfo
.ehash
=
2275 alloc_large_system_hash("TCP established",
2276 sizeof(struct inet_ehash_bucket
),
2278 (num_physpages
>= 128 * 1024) ?
2282 &tcp_hashinfo
.ehash_size
,
2285 tcp_hashinfo
.ehash_size
= (1 << tcp_hashinfo
.ehash_size
) >> 1;
2286 for (i
= 0; i
< (tcp_hashinfo
.ehash_size
<< 1); i
++) {
2287 rwlock_init(&tcp_hashinfo
.ehash
[i
].lock
);
2288 INIT_HLIST_HEAD(&tcp_hashinfo
.ehash
[i
].chain
);
2291 tcp_hashinfo
.bhash
=
2292 alloc_large_system_hash("TCP bind",
2293 sizeof(struct inet_bind_hashbucket
),
2294 tcp_hashinfo
.ehash_size
,
2295 (num_physpages
>= 128 * 1024) ?
2299 &tcp_hashinfo
.bhash_size
,
2302 tcp_hashinfo
.bhash_size
= 1 << tcp_hashinfo
.bhash_size
;
2303 for (i
= 0; i
< tcp_hashinfo
.bhash_size
; i
++) {
2304 spin_lock_init(&tcp_hashinfo
.bhash
[i
].lock
);
2305 INIT_HLIST_HEAD(&tcp_hashinfo
.bhash
[i
].chain
);
2308 /* Try to be a bit smarter and adjust defaults depending
2309 * on available memory.
2311 for (order
= 0; ((1 << order
) << PAGE_SHIFT
) <
2312 (tcp_hashinfo
.bhash_size
* sizeof(struct inet_bind_hashbucket
));
2316 sysctl_local_port_range
[0] = 32768;
2317 sysctl_local_port_range
[1] = 61000;
2318 sysctl_tcp_max_tw_buckets
= 180000;
2319 sysctl_tcp_max_orphans
= 4096 << (order
- 4);
2320 sysctl_max_syn_backlog
= 1024;
2321 } else if (order
< 3) {
2322 sysctl_local_port_range
[0] = 1024 * (3 - order
);
2323 sysctl_tcp_max_tw_buckets
>>= (3 - order
);
2324 sysctl_tcp_max_orphans
>>= (3 - order
);
2325 sysctl_max_syn_backlog
= 128;
2327 tcp_hashinfo
.port_rover
= sysctl_local_port_range
[0] - 1;
2329 sysctl_tcp_mem
[0] = 768 << order
;
2330 sysctl_tcp_mem
[1] = 1024 << order
;
2331 sysctl_tcp_mem
[2] = 1536 << order
;
2334 sysctl_tcp_wmem
[2] = 64 * 1024;
2335 sysctl_tcp_rmem
[0] = PAGE_SIZE
;
2336 sysctl_tcp_rmem
[1] = 43689;
2337 sysctl_tcp_rmem
[2] = 2 * 43689;
2340 printk(KERN_INFO
"TCP: Hash tables configured "
2341 "(established %d bind %d)\n",
2342 tcp_hashinfo
.ehash_size
<< 1, tcp_hashinfo
.bhash_size
);
2344 tcp_register_congestion_control(&tcp_reno
);
2347 EXPORT_SYMBOL(inet_csk_accept
);
2348 EXPORT_SYMBOL(tcp_close
);
2349 EXPORT_SYMBOL(tcp_destroy_sock
);
2350 EXPORT_SYMBOL(tcp_disconnect
);
2351 EXPORT_SYMBOL(tcp_getsockopt
);
2352 EXPORT_SYMBOL(tcp_ioctl
);
2353 EXPORT_SYMBOL(tcp_poll
);
2354 EXPORT_SYMBOL(tcp_read_sock
);
2355 EXPORT_SYMBOL(tcp_recvmsg
);
2356 EXPORT_SYMBOL(tcp_sendmsg
);
2357 EXPORT_SYMBOL(tcp_sendpage
);
2358 EXPORT_SYMBOL(tcp_setsockopt
);
2359 EXPORT_SYMBOL(tcp_shutdown
);
2360 EXPORT_SYMBOL(tcp_statistics
);