2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * Generic TIME_WAIT sockets functions
8 * From code orinally in TCP
11 #include <linux/kernel.h>
12 #include <linux/kmemcheck.h>
13 #include <linux/slab.h>
14 #include <linux/module.h>
15 #include <net/inet_hashtables.h>
16 #include <net/inet_timewait_sock.h>
21 * inet_twsk_unhash - unhash a timewait socket from established hash
22 * @tw: timewait socket
24 * unhash a timewait socket from established hash, if hashed.
25 * ehash lock must be held by caller.
26 * Returns 1 if caller should call inet_twsk_put() after lock release.
28 int inet_twsk_unhash(struct inet_timewait_sock
*tw
)
30 if (hlist_nulls_unhashed(&tw
->tw_node
))
33 hlist_nulls_del_rcu(&tw
->tw_node
);
34 sk_nulls_node_init(&tw
->tw_node
);
36 * We cannot call inet_twsk_put() ourself under lock,
37 * caller must call it for us.
43 * inet_twsk_bind_unhash - unhash a timewait socket from bind hash
44 * @tw: timewait socket
45 * @hashinfo: hashinfo pointer
47 * unhash a timewait socket from bind hash, if hashed.
48 * bind hash lock must be held by caller.
49 * Returns 1 if caller should call inet_twsk_put() after lock release.
51 int inet_twsk_bind_unhash(struct inet_timewait_sock
*tw
,
52 struct inet_hashinfo
*hashinfo
)
54 struct inet_bind_bucket
*tb
= tw
->tw_tb
;
59 __hlist_del(&tw
->tw_bind_node
);
61 inet_bind_bucket_destroy(hashinfo
->bind_bucket_cachep
, tb
);
63 * We cannot call inet_twsk_put() ourself under lock,
64 * caller must call it for us.
69 /* Must be called with locally disabled BHs. */
70 static void __inet_twsk_kill(struct inet_timewait_sock
*tw
,
71 struct inet_hashinfo
*hashinfo
)
73 struct inet_bind_hashbucket
*bhead
;
75 /* Unlink from established hashes. */
76 spinlock_t
*lock
= inet_ehash_lockp(hashinfo
, tw
->tw_hash
);
79 refcnt
= inet_twsk_unhash(tw
);
82 /* Disassociate with bind bucket. */
83 bhead
= &hashinfo
->bhash
[inet_bhashfn(twsk_net(tw
), tw
->tw_num
,
84 hashinfo
->bhash_size
)];
86 spin_lock(&bhead
->lock
);
87 refcnt
+= inet_twsk_bind_unhash(tw
, hashinfo
);
88 spin_unlock(&bhead
->lock
);
90 BUG_ON(refcnt
>= atomic_read(&tw
->tw_refcnt
));
91 atomic_sub(refcnt
, &tw
->tw_refcnt
);
94 void inet_twsk_free(struct inet_timewait_sock
*tw
)
96 struct module
*owner
= tw
->tw_prot
->owner
;
97 twsk_destructor((struct sock
*)tw
);
98 #ifdef SOCK_REFCNT_DEBUG
99 pr_debug("%s timewait_sock %p released\n", tw
->tw_prot
->name
, tw
);
101 release_net(twsk_net(tw
));
102 kmem_cache_free(tw
->tw_prot
->twsk_prot
->twsk_slab
, tw
);
106 void inet_twsk_put(struct inet_timewait_sock
*tw
)
108 if (atomic_dec_and_test(&tw
->tw_refcnt
))
111 EXPORT_SYMBOL_GPL(inet_twsk_put
);
113 static void inet_twsk_add_node_rcu(struct inet_timewait_sock
*tw
,
114 struct hlist_nulls_head
*list
)
116 hlist_nulls_add_head_rcu(&tw
->tw_node
, list
);
119 static void inet_twsk_add_bind_node(struct inet_timewait_sock
*tw
,
120 struct hlist_head
*list
)
122 hlist_add_head(&tw
->tw_bind_node
, list
);
126 * Enter the time wait state. This is called with locally disabled BH.
127 * Essentially we whip up a timewait bucket, copy the relevant info into it
128 * from the SK, and mess with hash chains and list linkage.
130 void __inet_twsk_hashdance(struct inet_timewait_sock
*tw
, struct sock
*sk
,
131 struct inet_hashinfo
*hashinfo
)
133 const struct inet_sock
*inet
= inet_sk(sk
);
134 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
135 struct inet_ehash_bucket
*ehead
= inet_ehash_bucket(hashinfo
, sk
->sk_hash
);
136 spinlock_t
*lock
= inet_ehash_lockp(hashinfo
, sk
->sk_hash
);
137 struct inet_bind_hashbucket
*bhead
;
138 /* Step 1: Put TW into bind hash. Original socket stays there too.
139 Note, that any socket with inet->num != 0 MUST be bound in
140 binding cache, even if it is closed.
142 bhead
= &hashinfo
->bhash
[inet_bhashfn(twsk_net(tw
), inet
->inet_num
,
143 hashinfo
->bhash_size
)];
144 spin_lock(&bhead
->lock
);
145 tw
->tw_tb
= icsk
->icsk_bind_hash
;
146 WARN_ON(!icsk
->icsk_bind_hash
);
147 inet_twsk_add_bind_node(tw
, &tw
->tw_tb
->owners
);
148 spin_unlock(&bhead
->lock
);
153 * Step 2: Hash TW into tcp ehash chain.
155 * - tw_refcnt is set to 3 because :
156 * - We have one reference from bhash chain.
157 * - We have one reference from ehash chain.
158 * We can use atomic_set() because prior spin_lock()/spin_unlock()
159 * committed into memory all tw fields.
161 atomic_set(&tw
->tw_refcnt
, 1 + 1 + 1);
162 inet_twsk_add_node_rcu(tw
, &ehead
->chain
);
164 /* Step 3: Remove SK from hash chain */
165 if (__sk_nulls_del_node_init_rcu(sk
))
166 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, -1);
170 EXPORT_SYMBOL_GPL(__inet_twsk_hashdance
);
172 struct inet_timewait_sock
*inet_twsk_alloc(const struct sock
*sk
, const int state
)
174 struct inet_timewait_sock
*tw
=
175 kmem_cache_alloc(sk
->sk_prot_creator
->twsk_prot
->twsk_slab
,
178 const struct inet_sock
*inet
= inet_sk(sk
);
180 kmemcheck_annotate_bitfield(tw
, flags
);
182 /* Give us an identity. */
183 tw
->tw_daddr
= inet
->inet_daddr
;
184 tw
->tw_rcv_saddr
= inet
->inet_rcv_saddr
;
185 tw
->tw_bound_dev_if
= sk
->sk_bound_dev_if
;
186 tw
->tw_tos
= inet
->tos
;
187 tw
->tw_num
= inet
->inet_num
;
188 tw
->tw_state
= TCP_TIME_WAIT
;
189 tw
->tw_substate
= state
;
190 tw
->tw_sport
= inet
->inet_sport
;
191 tw
->tw_dport
= inet
->inet_dport
;
192 tw
->tw_family
= sk
->sk_family
;
193 tw
->tw_reuse
= sk
->sk_reuse
;
194 tw
->tw_hash
= sk
->sk_hash
;
196 tw
->tw_transparent
= inet
->transparent
;
197 tw
->tw_prot
= sk
->sk_prot_creator
;
198 twsk_net_set(tw
, hold_net(sock_net(sk
)));
200 * Because we use RCU lookups, we should not set tw_refcnt
201 * to a non null value before everything is setup for this
204 atomic_set(&tw
->tw_refcnt
, 0);
205 inet_twsk_dead_node_init(tw
);
206 __module_get(tw
->tw_prot
->owner
);
211 EXPORT_SYMBOL_GPL(inet_twsk_alloc
);
213 /* Returns non-zero if quota exceeded. */
214 static int inet_twdr_do_twkill_work(struct inet_timewait_death_row
*twdr
,
217 struct inet_timewait_sock
*tw
;
221 /* NOTE: compare this to previous version where lock
222 * was released after detaching chain. It was racy,
223 * because tw buckets are scheduled in not serialized context
224 * in 2.3 (with netfilter), and with softnet it is common, because
225 * soft irqs are not sequenced.
230 inet_twsk_for_each_inmate(tw
, &twdr
->cells
[slot
]) {
231 __inet_twsk_del_dead_node(tw
);
232 spin_unlock(&twdr
->death_lock
);
233 __inet_twsk_kill(tw
, twdr
->hashinfo
);
235 NET_INC_STATS_BH(twsk_net(tw
), LINUX_MIB_TIMEWAITED
);
239 spin_lock(&twdr
->death_lock
);
240 if (killed
> INET_TWDR_TWKILL_QUOTA
) {
245 /* While we dropped twdr->death_lock, another cpu may have
246 * killed off the next TW bucket in the list, therefore
247 * do a fresh re-read of the hlist head node with the
248 * lock reacquired. We still use the hlist traversal
249 * macro in order to get the prefetches.
254 twdr
->tw_count
-= killed
;
255 #ifndef CONFIG_NET_NS
256 NET_ADD_STATS_BH(&init_net
, LINUX_MIB_TIMEWAITED
, killed
);
261 void inet_twdr_hangman(unsigned long data
)
263 struct inet_timewait_death_row
*twdr
;
264 unsigned int need_timer
;
266 twdr
= (struct inet_timewait_death_row
*)data
;
267 spin_lock(&twdr
->death_lock
);
269 if (twdr
->tw_count
== 0)
273 if (inet_twdr_do_twkill_work(twdr
, twdr
->slot
)) {
274 twdr
->thread_slots
|= (1 << twdr
->slot
);
275 schedule_work(&twdr
->twkill_work
);
278 /* We purged the entire slot, anything left? */
281 twdr
->slot
= ((twdr
->slot
+ 1) & (INET_TWDR_TWKILL_SLOTS
- 1));
284 mod_timer(&twdr
->tw_timer
, jiffies
+ twdr
->period
);
286 spin_unlock(&twdr
->death_lock
);
288 EXPORT_SYMBOL_GPL(inet_twdr_hangman
);
290 void inet_twdr_twkill_work(struct work_struct
*work
)
292 struct inet_timewait_death_row
*twdr
=
293 container_of(work
, struct inet_timewait_death_row
, twkill_work
);
296 BUILD_BUG_ON((INET_TWDR_TWKILL_SLOTS
- 1) >
297 (sizeof(twdr
->thread_slots
) * 8));
299 while (twdr
->thread_slots
) {
300 spin_lock_bh(&twdr
->death_lock
);
301 for (i
= 0; i
< INET_TWDR_TWKILL_SLOTS
; i
++) {
302 if (!(twdr
->thread_slots
& (1 << i
)))
305 while (inet_twdr_do_twkill_work(twdr
, i
) != 0) {
306 if (need_resched()) {
307 spin_unlock_bh(&twdr
->death_lock
);
309 spin_lock_bh(&twdr
->death_lock
);
313 twdr
->thread_slots
&= ~(1 << i
);
315 spin_unlock_bh(&twdr
->death_lock
);
318 EXPORT_SYMBOL_GPL(inet_twdr_twkill_work
);
320 /* These are always called from BH context. See callers in
321 * tcp_input.c to verify this.
324 /* This is for handling early-kills of TIME_WAIT sockets. */
325 void inet_twsk_deschedule(struct inet_timewait_sock
*tw
,
326 struct inet_timewait_death_row
*twdr
)
328 spin_lock(&twdr
->death_lock
);
329 if (inet_twsk_del_dead_node(tw
)) {
331 if (--twdr
->tw_count
== 0)
332 del_timer(&twdr
->tw_timer
);
334 spin_unlock(&twdr
->death_lock
);
335 __inet_twsk_kill(tw
, twdr
->hashinfo
);
337 EXPORT_SYMBOL(inet_twsk_deschedule
);
339 void inet_twsk_schedule(struct inet_timewait_sock
*tw
,
340 struct inet_timewait_death_row
*twdr
,
341 const int timeo
, const int timewait_len
)
343 struct hlist_head
*list
;
346 /* timeout := RTO * 3.5
348 * 3.5 = 1+2+0.5 to wait for two retransmits.
350 * RATIONALE: if FIN arrived and we entered TIME-WAIT state,
351 * our ACK acking that FIN can be lost. If N subsequent retransmitted
352 * FINs (or previous seqments) are lost (probability of such event
353 * is p^(N+1), where p is probability to lose single packet and
354 * time to detect the loss is about RTO*(2^N - 1) with exponential
355 * backoff). Normal timewait length is calculated so, that we
356 * waited at least for one retransmitted FIN (maximal RTO is 120sec).
357 * [ BTW Linux. following BSD, violates this requirement waiting
358 * only for 60sec, we should wait at least for 240 secs.
359 * Well, 240 consumes too much of resources 8)
361 * This interval is not reduced to catch old duplicate and
362 * responces to our wandering segments living for two MSLs.
363 * However, if we use PAWS to detect
364 * old duplicates, we can reduce the interval to bounds required
365 * by RTO, rather than MSL. So, if peer understands PAWS, we
366 * kill tw bucket after 3.5*RTO (it is important that this number
367 * is greater than TS tick!) and detect old duplicates with help
370 slot
= (timeo
+ (1 << INET_TWDR_RECYCLE_TICK
) - 1) >> INET_TWDR_RECYCLE_TICK
;
372 spin_lock(&twdr
->death_lock
);
374 /* Unlink it, if it was scheduled */
375 if (inet_twsk_del_dead_node(tw
))
378 atomic_inc(&tw
->tw_refcnt
);
380 if (slot
>= INET_TWDR_RECYCLE_SLOTS
) {
381 /* Schedule to slow timer */
382 if (timeo
>= timewait_len
) {
383 slot
= INET_TWDR_TWKILL_SLOTS
- 1;
385 slot
= DIV_ROUND_UP(timeo
, twdr
->period
);
386 if (slot
>= INET_TWDR_TWKILL_SLOTS
)
387 slot
= INET_TWDR_TWKILL_SLOTS
- 1;
389 tw
->tw_ttd
= inet_tw_time_stamp() + timeo
;
390 slot
= (twdr
->slot
+ slot
) & (INET_TWDR_TWKILL_SLOTS
- 1);
391 list
= &twdr
->cells
[slot
];
393 tw
->tw_ttd
= inet_tw_time_stamp() + (slot
<< INET_TWDR_RECYCLE_TICK
);
395 if (twdr
->twcal_hand
< 0) {
396 twdr
->twcal_hand
= 0;
397 twdr
->twcal_jiffie
= jiffies
;
398 twdr
->twcal_timer
.expires
= twdr
->twcal_jiffie
+
399 (slot
<< INET_TWDR_RECYCLE_TICK
);
400 add_timer(&twdr
->twcal_timer
);
402 if (time_after(twdr
->twcal_timer
.expires
,
403 jiffies
+ (slot
<< INET_TWDR_RECYCLE_TICK
)))
404 mod_timer(&twdr
->twcal_timer
,
405 jiffies
+ (slot
<< INET_TWDR_RECYCLE_TICK
));
406 slot
= (twdr
->twcal_hand
+ slot
) & (INET_TWDR_RECYCLE_SLOTS
- 1);
408 list
= &twdr
->twcal_row
[slot
];
411 hlist_add_head(&tw
->tw_death_node
, list
);
413 if (twdr
->tw_count
++ == 0)
414 mod_timer(&twdr
->tw_timer
, jiffies
+ twdr
->period
);
415 spin_unlock(&twdr
->death_lock
);
417 EXPORT_SYMBOL_GPL(inet_twsk_schedule
);
419 void inet_twdr_twcal_tick(unsigned long data
)
421 struct inet_timewait_death_row
*twdr
;
424 unsigned long now
= jiffies
;
428 twdr
= (struct inet_timewait_death_row
*)data
;
430 spin_lock(&twdr
->death_lock
);
431 if (twdr
->twcal_hand
< 0)
434 slot
= twdr
->twcal_hand
;
435 j
= twdr
->twcal_jiffie
;
437 for (n
= 0; n
< INET_TWDR_RECYCLE_SLOTS
; n
++) {
438 if (time_before_eq(j
, now
)) {
439 struct hlist_node
*safe
;
440 struct inet_timewait_sock
*tw
;
442 inet_twsk_for_each_inmate_safe(tw
, safe
,
443 &twdr
->twcal_row
[slot
]) {
444 __inet_twsk_del_dead_node(tw
);
445 __inet_twsk_kill(tw
, twdr
->hashinfo
);
447 NET_INC_STATS_BH(twsk_net(tw
), LINUX_MIB_TIMEWAITKILLED
);
455 twdr
->twcal_jiffie
= j
;
456 twdr
->twcal_hand
= slot
;
459 if (!hlist_empty(&twdr
->twcal_row
[slot
])) {
460 mod_timer(&twdr
->twcal_timer
, j
);
464 j
+= 1 << INET_TWDR_RECYCLE_TICK
;
465 slot
= (slot
+ 1) & (INET_TWDR_RECYCLE_SLOTS
- 1);
467 twdr
->twcal_hand
= -1;
470 if ((twdr
->tw_count
-= killed
) == 0)
471 del_timer(&twdr
->tw_timer
);
472 #ifndef CONFIG_NET_NS
473 NET_ADD_STATS_BH(&init_net
, LINUX_MIB_TIMEWAITKILLED
, killed
);
475 spin_unlock(&twdr
->death_lock
);
477 EXPORT_SYMBOL_GPL(inet_twdr_twcal_tick
);
479 void inet_twsk_purge(struct inet_hashinfo
*hashinfo
,
480 struct inet_timewait_death_row
*twdr
, int family
)
482 struct inet_timewait_sock
*tw
;
484 struct hlist_nulls_node
*node
;
487 for (slot
= 0; slot
<= hashinfo
->ehash_mask
; slot
++) {
488 struct inet_ehash_bucket
*head
= &hashinfo
->ehash
[slot
];
492 sk_nulls_for_each_rcu(sk
, node
, &head
->chain
) {
493 if (sk
->sk_state
!= TCP_TIME_WAIT
)
496 if ((tw
->tw_family
!= family
) ||
497 atomic_read(&twsk_net(tw
)->count
))
500 if (unlikely(!atomic_inc_not_zero(&tw
->tw_refcnt
)))
503 if (unlikely((tw
->tw_family
!= family
) ||
504 atomic_read(&twsk_net(tw
)->count
))) {
511 inet_twsk_deschedule(tw
, twdr
);
516 /* If the nulls value we got at the end of this lookup is
517 * not the expected one, we must restart lookup.
518 * We probably met an item that was moved to another chain.
520 if (get_nulls_value(node
) != slot
)
525 EXPORT_SYMBOL_GPL(inet_twsk_purge
);
This page took 0.048262 seconds and 5 git commands to generate.