Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * mm/thrash.c | |
3 | * | |
4 | * Copyright (C) 2004, Red Hat, Inc. | |
5 | * Copyright (C) 2004, Rik van Riel <riel@redhat.com> | |
6 | * Released under the GPL, see the file COPYING for details. | |
7 | * | |
8 | * Simple token based thrashing protection, using the algorithm | |
e21c7ffd | 9 | * described in: http://www.cse.ohio-state.edu/hpcs/WWW/HTML/publications/abs05-1.html |
7602bdf2 AC |
10 | * |
11 | * Sep 2006, Ashwin Chaugule <ashwin.chaugule@celunite.com> | |
12 | * Improved algorithm to pass token: | |
13 | * Each task has a priority which is incremented if it contended | |
14 | * for the token in an interval less than its previous attempt. | |
15 | * If the token is acquired, that task's priority is boosted to prevent | |
16 | * the token from bouncing around too often and to let the task make | |
17 | * some progress in its execution. | |
1da177e4 | 18 | */ |
7602bdf2 | 19 | |
1da177e4 LT |
20 | #include <linux/jiffies.h> |
21 | #include <linux/mm.h> | |
22 | #include <linux/sched.h> | |
23 | #include <linux/swap.h> | |
a433658c | 24 | #include <linux/memcontrol.h> |
1da177e4 | 25 | |
83cd81a3 KM |
26 | #include <trace/events/vmscan.h> |
27 | ||
d7911ef3 KM |
28 | #define TOKEN_AGING_INTERVAL (0xFF) |
29 | ||
1da177e4 | 30 | static DEFINE_SPINLOCK(swap_token_lock); |
7602bdf2 | 31 | struct mm_struct *swap_token_mm; |
a433658c | 32 | struct mem_cgroup *swap_token_memcg; |
1da177e4 | 33 | |
a433658c KM |
34 | #ifdef CONFIG_CGROUP_MEM_RES_CTLR |
35 | static struct mem_cgroup *swap_token_memcg_from_mm(struct mm_struct *mm) | |
36 | { | |
37 | struct mem_cgroup *memcg; | |
38 | ||
39 | memcg = try_get_mem_cgroup_from_mm(mm); | |
40 | if (memcg) | |
41 | css_put(mem_cgroup_css(memcg)); | |
42 | ||
43 | return memcg; | |
44 | } | |
45 | #else | |
46 | static struct mem_cgroup *swap_token_memcg_from_mm(struct mm_struct *mm) | |
47 | { | |
48 | return NULL; | |
49 | } | |
50 | #endif | |
51 | ||
a5c9b696 | 52 | void grab_swap_token(struct mm_struct *mm) |
1da177e4 | 53 | { |
7602bdf2 | 54 | int current_interval; |
83cd81a3 | 55 | unsigned int old_prio = mm->token_priority; |
53bb01f5 KM |
56 | static unsigned int global_faults; |
57 | static unsigned int last_aging; | |
1da177e4 | 58 | |
7602bdf2 | 59 | global_faults++; |
1da177e4 | 60 | |
a5c9b696 | 61 | current_interval = global_faults - mm->faultstamp; |
1da177e4 | 62 | |
7602bdf2 AC |
63 | if (!spin_trylock(&swap_token_lock)) |
64 | return; | |
1da177e4 | 65 | |
7602bdf2 | 66 | /* First come first served */ |
a433658c KM |
67 | if (!swap_token_mm) |
68 | goto replace_token; | |
69 | ||
d7911ef3 KM |
70 | if ((global_faults - last_aging) > TOKEN_AGING_INTERVAL) { |
71 | swap_token_mm->token_priority /= 2; | |
72 | last_aging = global_faults; | |
73 | } | |
74 | ||
a433658c KM |
75 | if (mm == swap_token_mm) { |
76 | mm->token_priority += 2; | |
83cd81a3 | 77 | goto update_priority; |
7602bdf2 | 78 | } |
1da177e4 | 79 | |
a433658c KM |
80 | if (current_interval < mm->last_interval) |
81 | mm->token_priority++; | |
82 | else { | |
83 | if (likely(mm->token_priority > 0)) | |
84 | mm->token_priority--; | |
1da177e4 | 85 | } |
7602bdf2 | 86 | |
a433658c KM |
87 | /* Check if we deserve the token */ |
88 | if (mm->token_priority > swap_token_mm->token_priority) | |
89 | goto replace_token; | |
90 | ||
83cd81a3 | 91 | update_priority: |
d7911ef3 | 92 | trace_update_swap_token_priority(mm, old_prio, swap_token_mm); |
83cd81a3 | 93 | |
7602bdf2 | 94 | out: |
a5c9b696 HD |
95 | mm->faultstamp = global_faults; |
96 | mm->last_interval = current_interval; | |
7602bdf2 | 97 | spin_unlock(&swap_token_lock); |
a433658c KM |
98 | return; |
99 | ||
100 | replace_token: | |
101 | mm->token_priority += 2; | |
83cd81a3 | 102 | trace_replace_swap_token(swap_token_mm, mm); |
a433658c KM |
103 | swap_token_mm = mm; |
104 | swap_token_memcg = swap_token_memcg_from_mm(mm); | |
d7911ef3 | 105 | last_aging = global_faults; |
a433658c | 106 | goto out; |
1da177e4 LT |
107 | } |
108 | ||
109 | /* Called on process exit. */ | |
110 | void __put_swap_token(struct mm_struct *mm) | |
111 | { | |
112 | spin_lock(&swap_token_lock); | |
a433658c | 113 | if (likely(mm == swap_token_mm)) { |
83cd81a3 | 114 | trace_put_swap_token(swap_token_mm); |
7602bdf2 | 115 | swap_token_mm = NULL; |
a433658c KM |
116 | swap_token_memcg = NULL; |
117 | } | |
1da177e4 LT |
118 | spin_unlock(&swap_token_lock); |
119 | } | |
a433658c KM |
120 | |
121 | static bool match_memcg(struct mem_cgroup *a, struct mem_cgroup *b) | |
122 | { | |
123 | if (!a) | |
124 | return true; | |
125 | if (!b) | |
126 | return true; | |
127 | if (a == b) | |
128 | return true; | |
129 | return false; | |
130 | } | |
131 | ||
132 | void disable_swap_token(struct mem_cgroup *memcg) | |
133 | { | |
134 | /* memcg reclaim don't disable unrelated mm token. */ | |
135 | if (match_memcg(memcg, swap_token_memcg)) { | |
136 | spin_lock(&swap_token_lock); | |
137 | if (match_memcg(memcg, swap_token_memcg)) { | |
83cd81a3 | 138 | trace_disable_swap_token(swap_token_mm); |
a433658c KM |
139 | swap_token_mm = NULL; |
140 | swap_token_memcg = NULL; | |
141 | } | |
142 | spin_unlock(&swap_token_lock); | |
143 | } | |
144 | } |