Commit | Line | Data |
---|---|---|
e7e1ef43 SS |
1 | /* |
2 | * Copyright (C) 2014 Sergey Senozhatsky. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation; either version | |
7 | * 2 of the License, or (at your option) any later version. | |
8 | */ | |
9 | ||
10 | #include <linux/kernel.h> | |
11 | #include <linux/string.h> | |
fcfa8d95 | 12 | #include <linux/err.h> |
e7e1ef43 SS |
13 | #include <linux/slab.h> |
14 | #include <linux/wait.h> | |
15 | #include <linux/sched.h> | |
16 | ||
17 | #include "zcomp.h" | |
18 | #include "zcomp_lzo.h" | |
6e76668e SS |
19 | #ifdef CONFIG_ZRAM_LZ4_COMPRESS |
20 | #include "zcomp_lz4.h" | |
21 | #endif | |
e7e1ef43 | 22 | |
9cc97529 SS |
23 | /* |
24 | * single zcomp_strm backend | |
25 | */ | |
26 | struct zcomp_strm_single { | |
27 | struct mutex strm_lock; | |
28 | struct zcomp_strm *zstrm; | |
29 | }; | |
30 | ||
beca3ec7 SS |
31 | /* |
32 | * multi zcomp_strm backend | |
33 | */ | |
34 | struct zcomp_strm_multi { | |
35 | /* protect strm list */ | |
36 | spinlock_t strm_lock; | |
37 | /* max possible number of zstrm streams */ | |
38 | int max_strm; | |
39 | /* number of available zstrm streams */ | |
40 | int avail_strm; | |
41 | /* list of available strms */ | |
42 | struct list_head idle_strm; | |
43 | wait_queue_head_t strm_wait; | |
44 | }; | |
45 | ||
e46b8a03 SS |
46 | static struct zcomp_backend *backends[] = { |
47 | &zcomp_lzo, | |
6e76668e SS |
48 | #ifdef CONFIG_ZRAM_LZ4_COMPRESS |
49 | &zcomp_lz4, | |
50 | #endif | |
e46b8a03 SS |
51 | NULL |
52 | }; | |
53 | ||
e7e1ef43 SS |
54 | static struct zcomp_backend *find_backend(const char *compress) |
55 | { | |
e46b8a03 SS |
56 | int i = 0; |
57 | while (backends[i]) { | |
58 | if (sysfs_streq(compress, backends[i]->name)) | |
59 | break; | |
60 | i++; | |
61 | } | |
62 | return backends[i]; | |
e7e1ef43 SS |
63 | } |
64 | ||
65 | static void zcomp_strm_free(struct zcomp *comp, struct zcomp_strm *zstrm) | |
66 | { | |
67 | if (zstrm->private) | |
68 | comp->backend->destroy(zstrm->private); | |
69 | free_pages((unsigned long)zstrm->buffer, 1); | |
70 | kfree(zstrm); | |
71 | } | |
72 | ||
73 | /* | |
74 | * allocate new zcomp_strm structure with ->private initialized by | |
75 | * backend, return NULL on error | |
76 | */ | |
75d8947a | 77 | static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp, gfp_t flags) |
e7e1ef43 | 78 | { |
75d8947a | 79 | struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), flags); |
e7e1ef43 SS |
80 | if (!zstrm) |
81 | return NULL; | |
82 | ||
75d8947a | 83 | zstrm->private = comp->backend->create(flags); |
e7e1ef43 SS |
84 | /* |
85 | * allocate 2 pages. 1 for compressed data, plus 1 extra for the | |
86 | * case when compressed size is larger than the original one | |
87 | */ | |
75d8947a | 88 | zstrm->buffer = (void *)__get_free_pages(flags | __GFP_ZERO, 1); |
e7e1ef43 SS |
89 | if (!zstrm->private || !zstrm->buffer) { |
90 | zcomp_strm_free(comp, zstrm); | |
91 | zstrm = NULL; | |
92 | } | |
93 | return zstrm; | |
94 | } | |
95 | ||
beca3ec7 SS |
96 | /* |
97 | * get idle zcomp_strm or wait until other process release | |
98 | * (zcomp_strm_release()) one for us | |
99 | */ | |
100 | static struct zcomp_strm *zcomp_strm_multi_find(struct zcomp *comp) | |
101 | { | |
102 | struct zcomp_strm_multi *zs = comp->stream; | |
103 | struct zcomp_strm *zstrm; | |
104 | ||
105 | while (1) { | |
106 | spin_lock(&zs->strm_lock); | |
107 | if (!list_empty(&zs->idle_strm)) { | |
108 | zstrm = list_entry(zs->idle_strm.next, | |
109 | struct zcomp_strm, list); | |
110 | list_del(&zstrm->list); | |
111 | spin_unlock(&zs->strm_lock); | |
112 | return zstrm; | |
113 | } | |
114 | /* zstrm streams limit reached, wait for idle stream */ | |
115 | if (zs->avail_strm >= zs->max_strm) { | |
116 | spin_unlock(&zs->strm_lock); | |
117 | wait_event(zs->strm_wait, !list_empty(&zs->idle_strm)); | |
118 | continue; | |
119 | } | |
120 | /* allocate new zstrm stream */ | |
121 | zs->avail_strm++; | |
122 | spin_unlock(&zs->strm_lock); | |
75d8947a MK |
123 | /* |
124 | * This function can be called in swapout/fs write path | |
125 | * so we can't use GFP_FS|IO. And it assumes we already | |
126 | * have at least one stream in zram initialization so we | |
127 | * don't do best effort to allocate more stream in here. | |
128 | * A default stream will work well without further multiple | |
129 | * streams. That's why we use NORETRY | NOWARN. | |
130 | */ | |
131 | zstrm = zcomp_strm_alloc(comp, GFP_NOIO | __GFP_NORETRY | | |
132 | __GFP_NOWARN); | |
beca3ec7 SS |
133 | if (!zstrm) { |
134 | spin_lock(&zs->strm_lock); | |
135 | zs->avail_strm--; | |
136 | spin_unlock(&zs->strm_lock); | |
137 | wait_event(zs->strm_wait, !list_empty(&zs->idle_strm)); | |
138 | continue; | |
139 | } | |
140 | break; | |
141 | } | |
142 | return zstrm; | |
143 | } | |
144 | ||
145 | /* add stream back to idle list and wake up waiter or free the stream */ | |
146 | static void zcomp_strm_multi_release(struct zcomp *comp, struct zcomp_strm *zstrm) | |
147 | { | |
148 | struct zcomp_strm_multi *zs = comp->stream; | |
149 | ||
150 | spin_lock(&zs->strm_lock); | |
151 | if (zs->avail_strm <= zs->max_strm) { | |
152 | list_add(&zstrm->list, &zs->idle_strm); | |
153 | spin_unlock(&zs->strm_lock); | |
154 | wake_up(&zs->strm_wait); | |
155 | return; | |
156 | } | |
157 | ||
158 | zs->avail_strm--; | |
159 | spin_unlock(&zs->strm_lock); | |
160 | zcomp_strm_free(comp, zstrm); | |
161 | } | |
162 | ||
fe8eb122 | 163 | /* change max_strm limit */ |
60a726e3 | 164 | static bool zcomp_strm_multi_set_max_streams(struct zcomp *comp, int num_strm) |
fe8eb122 SS |
165 | { |
166 | struct zcomp_strm_multi *zs = comp->stream; | |
167 | struct zcomp_strm *zstrm; | |
168 | ||
169 | spin_lock(&zs->strm_lock); | |
170 | zs->max_strm = num_strm; | |
171 | /* | |
172 | * if user has lowered the limit and there are idle streams, | |
173 | * immediately free as much streams (and memory) as we can. | |
174 | */ | |
175 | while (zs->avail_strm > num_strm && !list_empty(&zs->idle_strm)) { | |
176 | zstrm = list_entry(zs->idle_strm.next, | |
177 | struct zcomp_strm, list); | |
178 | list_del(&zstrm->list); | |
179 | zcomp_strm_free(comp, zstrm); | |
180 | zs->avail_strm--; | |
181 | } | |
182 | spin_unlock(&zs->strm_lock); | |
60a726e3 | 183 | return true; |
fe8eb122 SS |
184 | } |
185 | ||
beca3ec7 SS |
186 | static void zcomp_strm_multi_destroy(struct zcomp *comp) |
187 | { | |
188 | struct zcomp_strm_multi *zs = comp->stream; | |
189 | struct zcomp_strm *zstrm; | |
190 | ||
191 | while (!list_empty(&zs->idle_strm)) { | |
192 | zstrm = list_entry(zs->idle_strm.next, | |
193 | struct zcomp_strm, list); | |
194 | list_del(&zstrm->list); | |
195 | zcomp_strm_free(comp, zstrm); | |
196 | } | |
197 | kfree(zs); | |
198 | } | |
199 | ||
200 | static int zcomp_strm_multi_create(struct zcomp *comp, int max_strm) | |
201 | { | |
202 | struct zcomp_strm *zstrm; | |
203 | struct zcomp_strm_multi *zs; | |
204 | ||
205 | comp->destroy = zcomp_strm_multi_destroy; | |
206 | comp->strm_find = zcomp_strm_multi_find; | |
207 | comp->strm_release = zcomp_strm_multi_release; | |
fe8eb122 | 208 | comp->set_max_streams = zcomp_strm_multi_set_max_streams; |
beca3ec7 SS |
209 | zs = kmalloc(sizeof(struct zcomp_strm_multi), GFP_KERNEL); |
210 | if (!zs) | |
211 | return -ENOMEM; | |
212 | ||
213 | comp->stream = zs; | |
214 | spin_lock_init(&zs->strm_lock); | |
215 | INIT_LIST_HEAD(&zs->idle_strm); | |
216 | init_waitqueue_head(&zs->strm_wait); | |
217 | zs->max_strm = max_strm; | |
218 | zs->avail_strm = 1; | |
219 | ||
75d8947a | 220 | zstrm = zcomp_strm_alloc(comp, GFP_KERNEL); |
beca3ec7 SS |
221 | if (!zstrm) { |
222 | kfree(zs); | |
223 | return -ENOMEM; | |
224 | } | |
225 | list_add(&zstrm->list, &zs->idle_strm); | |
226 | return 0; | |
227 | } | |
228 | ||
9cc97529 SS |
229 | static struct zcomp_strm *zcomp_strm_single_find(struct zcomp *comp) |
230 | { | |
231 | struct zcomp_strm_single *zs = comp->stream; | |
232 | mutex_lock(&zs->strm_lock); | |
233 | return zs->zstrm; | |
234 | } | |
235 | ||
236 | static void zcomp_strm_single_release(struct zcomp *comp, | |
237 | struct zcomp_strm *zstrm) | |
238 | { | |
239 | struct zcomp_strm_single *zs = comp->stream; | |
240 | mutex_unlock(&zs->strm_lock); | |
241 | } | |
242 | ||
60a726e3 | 243 | static bool zcomp_strm_single_set_max_streams(struct zcomp *comp, int num_strm) |
fe8eb122 SS |
244 | { |
245 | /* zcomp_strm_single support only max_comp_streams == 1 */ | |
60a726e3 | 246 | return false; |
fe8eb122 SS |
247 | } |
248 | ||
9cc97529 SS |
249 | static void zcomp_strm_single_destroy(struct zcomp *comp) |
250 | { | |
251 | struct zcomp_strm_single *zs = comp->stream; | |
252 | zcomp_strm_free(comp, zs->zstrm); | |
253 | kfree(zs); | |
254 | } | |
255 | ||
256 | static int zcomp_strm_single_create(struct zcomp *comp) | |
257 | { | |
258 | struct zcomp_strm_single *zs; | |
259 | ||
260 | comp->destroy = zcomp_strm_single_destroy; | |
261 | comp->strm_find = zcomp_strm_single_find; | |
262 | comp->strm_release = zcomp_strm_single_release; | |
fe8eb122 | 263 | comp->set_max_streams = zcomp_strm_single_set_max_streams; |
9cc97529 SS |
264 | zs = kmalloc(sizeof(struct zcomp_strm_single), GFP_KERNEL); |
265 | if (!zs) | |
266 | return -ENOMEM; | |
267 | ||
268 | comp->stream = zs; | |
269 | mutex_init(&zs->strm_lock); | |
75d8947a | 270 | zs->zstrm = zcomp_strm_alloc(comp, GFP_KERNEL); |
9cc97529 SS |
271 | if (!zs->zstrm) { |
272 | kfree(zs); | |
273 | return -ENOMEM; | |
274 | } | |
275 | return 0; | |
276 | } | |
277 | ||
e46b8a03 SS |
278 | /* show available compressors */ |
279 | ssize_t zcomp_available_show(const char *comp, char *buf) | |
280 | { | |
281 | ssize_t sz = 0; | |
282 | int i = 0; | |
283 | ||
284 | while (backends[i]) { | |
4bbacd51 | 285 | if (!strcmp(comp, backends[i]->name)) |
56b4e8cb SS |
286 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, |
287 | "[%s] ", backends[i]->name); | |
e46b8a03 | 288 | else |
56b4e8cb SS |
289 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, |
290 | "%s ", backends[i]->name); | |
e46b8a03 SS |
291 | i++; |
292 | } | |
56b4e8cb | 293 | sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); |
e46b8a03 SS |
294 | return sz; |
295 | } | |
296 | ||
d93435c3 SS |
297 | bool zcomp_available_algorithm(const char *comp) |
298 | { | |
299 | return find_backend(comp) != NULL; | |
300 | } | |
301 | ||
60a726e3 | 302 | bool zcomp_set_max_streams(struct zcomp *comp, int num_strm) |
fe8eb122 SS |
303 | { |
304 | return comp->set_max_streams(comp, num_strm); | |
305 | } | |
306 | ||
e7e1ef43 SS |
307 | struct zcomp_strm *zcomp_strm_find(struct zcomp *comp) |
308 | { | |
9cc97529 | 309 | return comp->strm_find(comp); |
e7e1ef43 SS |
310 | } |
311 | ||
312 | void zcomp_strm_release(struct zcomp *comp, struct zcomp_strm *zstrm) | |
313 | { | |
9cc97529 | 314 | comp->strm_release(comp, zstrm); |
e7e1ef43 SS |
315 | } |
316 | ||
317 | int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, | |
318 | const unsigned char *src, size_t *dst_len) | |
319 | { | |
320 | return comp->backend->compress(src, zstrm->buffer, dst_len, | |
321 | zstrm->private); | |
322 | } | |
323 | ||
324 | int zcomp_decompress(struct zcomp *comp, const unsigned char *src, | |
325 | size_t src_len, unsigned char *dst) | |
326 | { | |
327 | return comp->backend->decompress(src, src_len, dst); | |
328 | } | |
329 | ||
330 | void zcomp_destroy(struct zcomp *comp) | |
331 | { | |
9cc97529 | 332 | comp->destroy(comp); |
e7e1ef43 SS |
333 | kfree(comp); |
334 | } | |
335 | ||
336 | /* | |
337 | * search available compressors for requested algorithm. | |
fcfa8d95 SS |
338 | * allocate new zcomp and initialize it. return compressing |
339 | * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL) | |
340 | * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in | |
3aaf14da LH |
341 | * case of allocation error, or any other error potentially |
342 | * returned by functions zcomp_strm_{multi,single}_create. | |
e7e1ef43 | 343 | */ |
beca3ec7 | 344 | struct zcomp *zcomp_create(const char *compress, int max_strm) |
e7e1ef43 SS |
345 | { |
346 | struct zcomp *comp; | |
347 | struct zcomp_backend *backend; | |
3aaf14da | 348 | int error; |
e7e1ef43 SS |
349 | |
350 | backend = find_backend(compress); | |
351 | if (!backend) | |
fcfa8d95 | 352 | return ERR_PTR(-EINVAL); |
e7e1ef43 SS |
353 | |
354 | comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL); | |
355 | if (!comp) | |
fcfa8d95 | 356 | return ERR_PTR(-ENOMEM); |
e7e1ef43 SS |
357 | |
358 | comp->backend = backend; | |
beca3ec7 | 359 | if (max_strm > 1) |
3aaf14da | 360 | error = zcomp_strm_multi_create(comp, max_strm); |
beca3ec7 | 361 | else |
3aaf14da LH |
362 | error = zcomp_strm_single_create(comp); |
363 | if (error) { | |
e7e1ef43 | 364 | kfree(comp); |
3aaf14da | 365 | return ERR_PTR(error); |
e7e1ef43 SS |
366 | } |
367 | return comp; | |
368 | } |