Commit | Line | Data |
---|---|---|
1403b1a3 PN |
1 | /* |
2 | * Copyright (c) Red Hat Inc. | |
3 | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sub license, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the | |
12 | * next paragraph) shall be included in all copies or substantial portions | |
13 | * of the Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | |
21 | * DEALINGS IN THE SOFTWARE. | |
22 | * | |
23 | * Authors: Dave Airlie <airlied@redhat.com> | |
24 | * Jerome Glisse <jglisse@redhat.com> | |
25 | * Pauli Nieminen <suokkos@gmail.com> | |
26 | */ | |
27 | ||
28 | /* simple list based uncached page pool | |
29 | * - Pool collects resently freed pages for reuse | |
30 | * - Use page->lru to keep a free list | |
31 | * - doesn't track currently in use pages | |
32 | */ | |
25d0479a JP |
33 | |
34 | #define pr_fmt(fmt) "[TTM] " fmt | |
35 | ||
1403b1a3 PN |
36 | #include <linux/list.h> |
37 | #include <linux/spinlock.h> | |
38 | #include <linux/highmem.h> | |
39 | #include <linux/mm_types.h> | |
07458661 | 40 | #include <linux/module.h> |
1403b1a3 | 41 | #include <linux/mm.h> |
4cdc840a | 42 | #include <linux/seq_file.h> /* for seq_printf */ |
2125b8a4 | 43 | #include <linux/slab.h> |
f9820a46 | 44 | #include <linux/dma-mapping.h> |
1403b1a3 | 45 | |
60063497 | 46 | #include <linux/atomic.h> |
1403b1a3 | 47 | |
760285e7 DH |
48 | #include <drm/ttm/ttm_bo_driver.h> |
49 | #include <drm/ttm/ttm_page_alloc.h> | |
1403b1a3 | 50 | |
d6678651 LT |
51 | #ifdef TTM_HAS_AGP |
52 | #include <asm/agp.h> | |
53 | #endif | |
1403b1a3 PN |
54 | |
55 | #define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) | |
56 | #define SMALL_ALLOCATION 16 | |
57 | #define FREE_ALL_PAGES (~0U) | |
58 | /* times are in msecs */ | |
59 | #define PAGE_FREE_INTERVAL 1000 | |
60 | ||
61 | /** | |
62 | * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. | |
63 | * | |
64 | * @lock: Protects the shared pool from concurrnet access. Must be used with | |
65 | * irqsave/irqrestore variants because pool allocator maybe called from | |
66 | * delayed work. | |
67 | * @fill_lock: Prevent concurrent calls to fill. | |
68 | * @list: Pool of free uc/wc pages for fast reuse. | |
69 | * @gfp_flags: Flags to pass for alloc_page. | |
70 | * @npages: Number of pages in pool. | |
71 | */ | |
72 | struct ttm_page_pool { | |
73 | spinlock_t lock; | |
74 | bool fill_lock; | |
75 | struct list_head list; | |
0e57a3cc | 76 | gfp_t gfp_flags; |
1403b1a3 | 77 | unsigned npages; |
07458661 PN |
78 | char *name; |
79 | unsigned long nfrees; | |
80 | unsigned long nrefills; | |
1403b1a3 PN |
81 | }; |
82 | ||
c96af79e PN |
83 | /** |
84 | * Limits for the pool. They are handled without locks because only place where | |
85 | * they may change is in sysfs store. They won't have immediate effect anyway | |
4abe4389 | 86 | * so forcing serialization to access them is pointless. |
c96af79e PN |
87 | */ |
88 | ||
1403b1a3 PN |
89 | struct ttm_pool_opts { |
90 | unsigned alloc_size; | |
91 | unsigned max_size; | |
92 | unsigned small; | |
93 | }; | |
94 | ||
95 | #define NUM_POOLS 4 | |
96 | ||
97 | /** | |
98 | * struct ttm_pool_manager - Holds memory pools for fst allocation | |
99 | * | |
100 | * Manager is read only object for pool code so it doesn't need locking. | |
101 | * | |
102 | * @free_interval: minimum number of jiffies between freeing pages from pool. | |
103 | * @page_alloc_inited: reference counting for pool allocation. | |
104 | * @work: Work that is used to shrink the pool. Work is only run when there is | |
105 | * some pages to free. | |
106 | * @small_allocation: Limit in number of pages what is small allocation. | |
107 | * | |
108 | * @pools: All pool objects in use. | |
109 | **/ | |
110 | struct ttm_pool_manager { | |
c96af79e | 111 | struct kobject kobj; |
1403b1a3 | 112 | struct shrinker mm_shrink; |
1403b1a3 PN |
113 | struct ttm_pool_opts options; |
114 | ||
115 | union { | |
116 | struct ttm_page_pool pools[NUM_POOLS]; | |
117 | struct { | |
118 | struct ttm_page_pool wc_pool; | |
119 | struct ttm_page_pool uc_pool; | |
120 | struct ttm_page_pool wc_pool_dma32; | |
121 | struct ttm_page_pool uc_pool_dma32; | |
122 | } ; | |
123 | }; | |
124 | }; | |
125 | ||
c96af79e PN |
126 | static struct attribute ttm_page_pool_max = { |
127 | .name = "pool_max_size", | |
128 | .mode = S_IRUGO | S_IWUSR | |
129 | }; | |
130 | static struct attribute ttm_page_pool_small = { | |
131 | .name = "pool_small_allocation", | |
132 | .mode = S_IRUGO | S_IWUSR | |
133 | }; | |
134 | static struct attribute ttm_page_pool_alloc_size = { | |
135 | .name = "pool_allocation_size", | |
136 | .mode = S_IRUGO | S_IWUSR | |
137 | }; | |
138 | ||
139 | static struct attribute *ttm_pool_attrs[] = { | |
140 | &ttm_page_pool_max, | |
141 | &ttm_page_pool_small, | |
142 | &ttm_page_pool_alloc_size, | |
143 | NULL | |
144 | }; | |
145 | ||
146 | static void ttm_pool_kobj_release(struct kobject *kobj) | |
147 | { | |
148 | struct ttm_pool_manager *m = | |
149 | container_of(kobj, struct ttm_pool_manager, kobj); | |
5870a4d9 | 150 | kfree(m); |
c96af79e PN |
151 | } |
152 | ||
153 | static ssize_t ttm_pool_store(struct kobject *kobj, | |
154 | struct attribute *attr, const char *buffer, size_t size) | |
155 | { | |
156 | struct ttm_pool_manager *m = | |
157 | container_of(kobj, struct ttm_pool_manager, kobj); | |
158 | int chars; | |
159 | unsigned val; | |
160 | chars = sscanf(buffer, "%u", &val); | |
161 | if (chars == 0) | |
162 | return size; | |
163 | ||
164 | /* Convert kb to number of pages */ | |
165 | val = val / (PAGE_SIZE >> 10); | |
166 | ||
167 | if (attr == &ttm_page_pool_max) | |
168 | m->options.max_size = val; | |
169 | else if (attr == &ttm_page_pool_small) | |
170 | m->options.small = val; | |
171 | else if (attr == &ttm_page_pool_alloc_size) { | |
172 | if (val > NUM_PAGES_TO_ALLOC*8) { | |
25d0479a | 173 | pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", |
4abe4389 TH |
174 | NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), |
175 | NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); | |
c96af79e PN |
176 | return size; |
177 | } else if (val > NUM_PAGES_TO_ALLOC) { | |
25d0479a JP |
178 | pr_warn("Setting allocation size to larger than %lu is not recommended\n", |
179 | NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); | |
c96af79e PN |
180 | } |
181 | m->options.alloc_size = val; | |
182 | } | |
183 | ||
184 | return size; | |
185 | } | |
186 | ||
187 | static ssize_t ttm_pool_show(struct kobject *kobj, | |
188 | struct attribute *attr, char *buffer) | |
189 | { | |
190 | struct ttm_pool_manager *m = | |
191 | container_of(kobj, struct ttm_pool_manager, kobj); | |
192 | unsigned val = 0; | |
193 | ||
194 | if (attr == &ttm_page_pool_max) | |
195 | val = m->options.max_size; | |
196 | else if (attr == &ttm_page_pool_small) | |
197 | val = m->options.small; | |
198 | else if (attr == &ttm_page_pool_alloc_size) | |
199 | val = m->options.alloc_size; | |
200 | ||
201 | val = val * (PAGE_SIZE >> 10); | |
202 | ||
203 | return snprintf(buffer, PAGE_SIZE, "%u\n", val); | |
204 | } | |
205 | ||
206 | static const struct sysfs_ops ttm_pool_sysfs_ops = { | |
207 | .show = &ttm_pool_show, | |
208 | .store = &ttm_pool_store, | |
209 | }; | |
210 | ||
211 | static struct kobj_type ttm_pool_kobj_type = { | |
212 | .release = &ttm_pool_kobj_release, | |
213 | .sysfs_ops = &ttm_pool_sysfs_ops, | |
214 | .default_attrs = ttm_pool_attrs, | |
215 | }; | |
216 | ||
5870a4d9 | 217 | static struct ttm_pool_manager *_manager; |
1403b1a3 | 218 | |
975efdb1 | 219 | #ifndef CONFIG_X86 |
1403b1a3 PN |
220 | static int set_pages_array_wb(struct page **pages, int addrinarray) |
221 | { | |
222 | #ifdef TTM_HAS_AGP | |
223 | int i; | |
224 | ||
225 | for (i = 0; i < addrinarray; i++) | |
226 | unmap_page_from_agp(pages[i]); | |
227 | #endif | |
228 | return 0; | |
229 | } | |
230 | ||
231 | static int set_pages_array_wc(struct page **pages, int addrinarray) | |
232 | { | |
233 | #ifdef TTM_HAS_AGP | |
234 | int i; | |
235 | ||
236 | for (i = 0; i < addrinarray; i++) | |
237 | map_page_into_agp(pages[i]); | |
238 | #endif | |
239 | return 0; | |
240 | } | |
241 | ||
242 | static int set_pages_array_uc(struct page **pages, int addrinarray) | |
243 | { | |
244 | #ifdef TTM_HAS_AGP | |
245 | int i; | |
246 | ||
247 | for (i = 0; i < addrinarray; i++) | |
248 | map_page_into_agp(pages[i]); | |
249 | #endif | |
250 | return 0; | |
251 | } | |
252 | #endif | |
253 | ||
254 | /** | |
255 | * Select the right pool or requested caching state and ttm flags. */ | |
256 | static struct ttm_page_pool *ttm_get_pool(int flags, | |
257 | enum ttm_caching_state cstate) | |
258 | { | |
259 | int pool_index; | |
260 | ||
261 | if (cstate == tt_cached) | |
262 | return NULL; | |
263 | ||
264 | if (cstate == tt_wc) | |
265 | pool_index = 0x0; | |
266 | else | |
267 | pool_index = 0x1; | |
268 | ||
269 | if (flags & TTM_PAGE_FLAG_DMA32) | |
270 | pool_index |= 0x2; | |
271 | ||
5870a4d9 | 272 | return &_manager->pools[pool_index]; |
1403b1a3 PN |
273 | } |
274 | ||
275 | /* set memory back to wb and free the pages. */ | |
276 | static void ttm_pages_put(struct page *pages[], unsigned npages) | |
277 | { | |
278 | unsigned i; | |
279 | if (set_pages_array_wb(pages, npages)) | |
25d0479a | 280 | pr_err("Failed to set %d pages to wb!\n", npages); |
1403b1a3 PN |
281 | for (i = 0; i < npages; ++i) |
282 | __free_page(pages[i]); | |
283 | } | |
284 | ||
285 | static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, | |
286 | unsigned freed_pages) | |
287 | { | |
288 | pool->npages -= freed_pages; | |
07458661 | 289 | pool->nfrees += freed_pages; |
1403b1a3 PN |
290 | } |
291 | ||
292 | /** | |
293 | * Free pages from pool. | |
294 | * | |
295 | * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC | |
296 | * number of pages in one go. | |
297 | * | |
298 | * @pool: to free the pages from | |
299 | * @free_all: If set to true will free all pages in pool | |
881fdaa5 | 300 | * @use_static: Safe to use static buffer |
1403b1a3 | 301 | **/ |
a91576d7 | 302 | static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free, |
881fdaa5 | 303 | bool use_static) |
1403b1a3 | 304 | { |
881fdaa5 | 305 | static struct page *static_buf[NUM_PAGES_TO_ALLOC]; |
1403b1a3 PN |
306 | unsigned long irq_flags; |
307 | struct page *p; | |
308 | struct page **pages_to_free; | |
309 | unsigned freed_pages = 0, | |
310 | npages_to_free = nr_free; | |
311 | ||
312 | if (NUM_PAGES_TO_ALLOC < nr_free) | |
313 | npages_to_free = NUM_PAGES_TO_ALLOC; | |
314 | ||
881fdaa5 TH |
315 | if (use_static) |
316 | pages_to_free = static_buf; | |
317 | else | |
318 | pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), | |
319 | GFP_KERNEL); | |
1403b1a3 | 320 | if (!pages_to_free) { |
25d0479a | 321 | pr_err("Failed to allocate memory for pool free operation\n"); |
1403b1a3 PN |
322 | return 0; |
323 | } | |
324 | ||
325 | restart: | |
326 | spin_lock_irqsave(&pool->lock, irq_flags); | |
327 | ||
328 | list_for_each_entry_reverse(p, &pool->list, lru) { | |
329 | if (freed_pages >= npages_to_free) | |
330 | break; | |
331 | ||
332 | pages_to_free[freed_pages++] = p; | |
333 | /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ | |
334 | if (freed_pages >= NUM_PAGES_TO_ALLOC) { | |
335 | /* remove range of pages from the pool */ | |
336 | __list_del(p->lru.prev, &pool->list); | |
337 | ||
338 | ttm_pool_update_free_locked(pool, freed_pages); | |
339 | /** | |
340 | * Because changing page caching is costly | |
341 | * we unlock the pool to prevent stalling. | |
342 | */ | |
343 | spin_unlock_irqrestore(&pool->lock, irq_flags); | |
344 | ||
345 | ttm_pages_put(pages_to_free, freed_pages); | |
346 | if (likely(nr_free != FREE_ALL_PAGES)) | |
347 | nr_free -= freed_pages; | |
348 | ||
349 | if (NUM_PAGES_TO_ALLOC >= nr_free) | |
350 | npages_to_free = nr_free; | |
351 | else | |
352 | npages_to_free = NUM_PAGES_TO_ALLOC; | |
353 | ||
354 | freed_pages = 0; | |
355 | ||
356 | /* free all so restart the processing */ | |
357 | if (nr_free) | |
358 | goto restart; | |
359 | ||
0d74f86f | 360 | /* Not allowed to fall through or break because |
1403b1a3 PN |
361 | * following context is inside spinlock while we are |
362 | * outside here. | |
363 | */ | |
364 | goto out; | |
365 | ||
366 | } | |
367 | } | |
368 | ||
1403b1a3 PN |
369 | /* remove range of pages from the pool */ |
370 | if (freed_pages) { | |
371 | __list_del(&p->lru, &pool->list); | |
372 | ||
373 | ttm_pool_update_free_locked(pool, freed_pages); | |
374 | nr_free -= freed_pages; | |
375 | } | |
376 | ||
377 | spin_unlock_irqrestore(&pool->lock, irq_flags); | |
378 | ||
379 | if (freed_pages) | |
380 | ttm_pages_put(pages_to_free, freed_pages); | |
381 | out: | |
881fdaa5 TH |
382 | if (pages_to_free != static_buf) |
383 | kfree(pages_to_free); | |
1403b1a3 PN |
384 | return nr_free; |
385 | } | |
386 | ||
1403b1a3 | 387 | /** |
4abe4389 | 388 | * Callback for mm to request pool to reduce number of page held. |
7dc19d5a DC |
389 | * |
390 | * XXX: (dchinner) Deadlock warning! | |
391 | * | |
7dc19d5a | 392 | * This code is crying out for a shrinker per pool.... |
1403b1a3 | 393 | */ |
7dc19d5a DC |
394 | static unsigned long |
395 | ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) | |
1403b1a3 | 396 | { |
71336e01 TH |
397 | static DEFINE_MUTEX(lock); |
398 | static unsigned start_pool; | |
1403b1a3 | 399 | unsigned i; |
71336e01 | 400 | unsigned pool_offset; |
1403b1a3 | 401 | struct ttm_page_pool *pool; |
1495f230 | 402 | int shrink_pages = sc->nr_to_scan; |
7dc19d5a | 403 | unsigned long freed = 0; |
1403b1a3 | 404 | |
71336e01 TH |
405 | if (!mutex_trylock(&lock)) |
406 | return SHRINK_STOP; | |
407 | pool_offset = ++start_pool % NUM_POOLS; | |
1403b1a3 PN |
408 | /* select start pool in round robin fashion */ |
409 | for (i = 0; i < NUM_POOLS; ++i) { | |
410 | unsigned nr_free = shrink_pages; | |
411 | if (shrink_pages == 0) | |
412 | break; | |
5870a4d9 | 413 | pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; |
881fdaa5 TH |
414 | /* OK to use static buffer since global mutex is held. */ |
415 | shrink_pages = ttm_page_pool_free(pool, nr_free, true); | |
7dc19d5a | 416 | freed += nr_free - shrink_pages; |
1403b1a3 | 417 | } |
71336e01 | 418 | mutex_unlock(&lock); |
7dc19d5a DC |
419 | return freed; |
420 | } | |
421 | ||
422 | ||
423 | static unsigned long | |
424 | ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) | |
425 | { | |
426 | unsigned i; | |
427 | unsigned long count = 0; | |
428 | ||
429 | for (i = 0; i < NUM_POOLS; ++i) | |
430 | count += _manager->pools[i].npages; | |
431 | ||
432 | return count; | |
1403b1a3 PN |
433 | } |
434 | ||
435 | static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) | |
436 | { | |
7dc19d5a DC |
437 | manager->mm_shrink.count_objects = ttm_pool_shrink_count; |
438 | manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; | |
1403b1a3 PN |
439 | manager->mm_shrink.seeks = 1; |
440 | register_shrinker(&manager->mm_shrink); | |
441 | } | |
442 | ||
443 | static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) | |
444 | { | |
445 | unregister_shrinker(&manager->mm_shrink); | |
446 | } | |
447 | ||
448 | static int ttm_set_pages_caching(struct page **pages, | |
449 | enum ttm_caching_state cstate, unsigned cpages) | |
450 | { | |
451 | int r = 0; | |
452 | /* Set page caching */ | |
453 | switch (cstate) { | |
454 | case tt_uncached: | |
455 | r = set_pages_array_uc(pages, cpages); | |
456 | if (r) | |
25d0479a | 457 | pr_err("Failed to set %d pages to uc!\n", cpages); |
1403b1a3 PN |
458 | break; |
459 | case tt_wc: | |
460 | r = set_pages_array_wc(pages, cpages); | |
461 | if (r) | |
25d0479a | 462 | pr_err("Failed to set %d pages to wc!\n", cpages); |
1403b1a3 PN |
463 | break; |
464 | default: | |
465 | break; | |
466 | } | |
467 | return r; | |
468 | } | |
469 | ||
470 | /** | |
471 | * Free pages the pages that failed to change the caching state. If there is | |
472 | * any pages that have changed their caching state already put them to the | |
473 | * pool. | |
474 | */ | |
475 | static void ttm_handle_caching_state_failure(struct list_head *pages, | |
476 | int ttm_flags, enum ttm_caching_state cstate, | |
477 | struct page **failed_pages, unsigned cpages) | |
478 | { | |
479 | unsigned i; | |
4abe4389 | 480 | /* Failed pages have to be freed */ |
1403b1a3 PN |
481 | for (i = 0; i < cpages; ++i) { |
482 | list_del(&failed_pages[i]->lru); | |
483 | __free_page(failed_pages[i]); | |
484 | } | |
485 | } | |
486 | ||
487 | /** | |
488 | * Allocate new pages with correct caching. | |
489 | * | |
490 | * This function is reentrant if caller updates count depending on number of | |
491 | * pages returned in pages array. | |
492 | */ | |
0e57a3cc | 493 | static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, |
1403b1a3 PN |
494 | int ttm_flags, enum ttm_caching_state cstate, unsigned count) |
495 | { | |
496 | struct page **caching_array; | |
497 | struct page *p; | |
498 | int r = 0; | |
499 | unsigned i, cpages; | |
500 | unsigned max_cpages = min(count, | |
501 | (unsigned)(PAGE_SIZE/sizeof(struct page *))); | |
502 | ||
503 | /* allocate array for page caching change */ | |
504 | caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); | |
505 | ||
506 | if (!caching_array) { | |
25d0479a | 507 | pr_err("Unable to allocate table for new pages\n"); |
1403b1a3 PN |
508 | return -ENOMEM; |
509 | } | |
510 | ||
511 | for (i = 0, cpages = 0; i < count; ++i) { | |
512 | p = alloc_page(gfp_flags); | |
513 | ||
514 | if (!p) { | |
25d0479a | 515 | pr_err("Unable to get page %u\n", i); |
1403b1a3 PN |
516 | |
517 | /* store already allocated pages in the pool after | |
518 | * setting the caching state */ | |
519 | if (cpages) { | |
4abe4389 TH |
520 | r = ttm_set_pages_caching(caching_array, |
521 | cstate, cpages); | |
1403b1a3 PN |
522 | if (r) |
523 | ttm_handle_caching_state_failure(pages, | |
524 | ttm_flags, cstate, | |
525 | caching_array, cpages); | |
526 | } | |
527 | r = -ENOMEM; | |
528 | goto out; | |
529 | } | |
530 | ||
531 | #ifdef CONFIG_HIGHMEM | |
532 | /* gfp flags of highmem page should never be dma32 so we | |
533 | * we should be fine in such case | |
534 | */ | |
535 | if (!PageHighMem(p)) | |
536 | #endif | |
537 | { | |
538 | caching_array[cpages++] = p; | |
539 | if (cpages == max_cpages) { | |
540 | ||
541 | r = ttm_set_pages_caching(caching_array, | |
542 | cstate, cpages); | |
543 | if (r) { | |
544 | ttm_handle_caching_state_failure(pages, | |
545 | ttm_flags, cstate, | |
546 | caching_array, cpages); | |
547 | goto out; | |
548 | } | |
549 | cpages = 0; | |
550 | } | |
551 | } | |
552 | ||
553 | list_add(&p->lru, pages); | |
554 | } | |
555 | ||
556 | if (cpages) { | |
557 | r = ttm_set_pages_caching(caching_array, cstate, cpages); | |
558 | if (r) | |
559 | ttm_handle_caching_state_failure(pages, | |
560 | ttm_flags, cstate, | |
561 | caching_array, cpages); | |
562 | } | |
563 | out: | |
564 | kfree(caching_array); | |
565 | ||
566 | return r; | |
567 | } | |
568 | ||
569 | /** | |
0d74f86f | 570 | * Fill the given pool if there aren't enough pages and the requested number of |
1403b1a3 PN |
571 | * pages is small. |
572 | */ | |
573 | static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, | |
574 | int ttm_flags, enum ttm_caching_state cstate, unsigned count, | |
575 | unsigned long *irq_flags) | |
576 | { | |
577 | struct page *p; | |
578 | int r; | |
579 | unsigned cpages = 0; | |
580 | /** | |
581 | * Only allow one pool fill operation at a time. | |
582 | * If pool doesn't have enough pages for the allocation new pages are | |
583 | * allocated from outside of pool. | |
584 | */ | |
585 | if (pool->fill_lock) | |
586 | return; | |
587 | ||
588 | pool->fill_lock = true; | |
589 | ||
0d74f86f KRW |
590 | /* If allocation request is small and there are not enough |
591 | * pages in a pool we fill the pool up first. */ | |
5870a4d9 | 592 | if (count < _manager->options.small |
1403b1a3 PN |
593 | && count > pool->npages) { |
594 | struct list_head new_pages; | |
5870a4d9 | 595 | unsigned alloc_size = _manager->options.alloc_size; |
1403b1a3 PN |
596 | |
597 | /** | |
598 | * Can't change page caching if in irqsave context. We have to | |
599 | * drop the pool->lock. | |
600 | */ | |
601 | spin_unlock_irqrestore(&pool->lock, *irq_flags); | |
602 | ||
603 | INIT_LIST_HEAD(&new_pages); | |
604 | r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, | |
605 | cstate, alloc_size); | |
606 | spin_lock_irqsave(&pool->lock, *irq_flags); | |
607 | ||
608 | if (!r) { | |
609 | list_splice(&new_pages, &pool->list); | |
07458661 | 610 | ++pool->nrefills; |
1403b1a3 PN |
611 | pool->npages += alloc_size; |
612 | } else { | |
25d0479a | 613 | pr_err("Failed to fill pool (%p)\n", pool); |
1403b1a3 PN |
614 | /* If we have any pages left put them to the pool. */ |
615 | list_for_each_entry(p, &pool->list, lru) { | |
616 | ++cpages; | |
617 | } | |
618 | list_splice(&new_pages, &pool->list); | |
619 | pool->npages += cpages; | |
620 | } | |
621 | ||
622 | } | |
623 | pool->fill_lock = false; | |
624 | } | |
625 | ||
626 | /** | |
0d74f86f | 627 | * Cut 'count' number of pages from the pool and put them on the return list. |
1403b1a3 | 628 | * |
0d74f86f | 629 | * @return count of pages still required to fulfill the request. |
1403b1a3 PN |
630 | */ |
631 | static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, | |
822c4d9a JG |
632 | struct list_head *pages, |
633 | int ttm_flags, | |
634 | enum ttm_caching_state cstate, | |
635 | unsigned count) | |
1403b1a3 PN |
636 | { |
637 | unsigned long irq_flags; | |
638 | struct list_head *p; | |
639 | unsigned i; | |
640 | ||
641 | spin_lock_irqsave(&pool->lock, irq_flags); | |
642 | ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags); | |
643 | ||
644 | if (count >= pool->npages) { | |
645 | /* take all pages from the pool */ | |
646 | list_splice_init(&pool->list, pages); | |
647 | count -= pool->npages; | |
648 | pool->npages = 0; | |
649 | goto out; | |
650 | } | |
651 | /* find the last pages to include for requested number of pages. Split | |
0d74f86f | 652 | * pool to begin and halve it to reduce search space. */ |
1403b1a3 PN |
653 | if (count <= pool->npages/2) { |
654 | i = 0; | |
655 | list_for_each(p, &pool->list) { | |
656 | if (++i == count) | |
657 | break; | |
658 | } | |
659 | } else { | |
660 | i = pool->npages + 1; | |
661 | list_for_each_prev(p, &pool->list) { | |
662 | if (--i == count) | |
663 | break; | |
664 | } | |
665 | } | |
0d74f86f | 666 | /* Cut 'count' number of pages from the pool */ |
1403b1a3 PN |
667 | list_cut_position(pages, &pool->list, p); |
668 | pool->npages -= count; | |
669 | count = 0; | |
670 | out: | |
671 | spin_unlock_irqrestore(&pool->lock, irq_flags); | |
672 | return count; | |
673 | } | |
674 | ||
8e7e7052 JG |
675 | /* Put all pages in pages list to correct pool to wait for reuse */ |
676 | static void ttm_put_pages(struct page **pages, unsigned npages, int flags, | |
677 | enum ttm_caching_state cstate) | |
678 | { | |
679 | unsigned long irq_flags; | |
680 | struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); | |
681 | unsigned i; | |
682 | ||
683 | if (pool == NULL) { | |
684 | /* No pool for this memory type so free the pages */ | |
685 | for (i = 0; i < npages; i++) { | |
686 | if (pages[i]) { | |
687 | if (page_count(pages[i]) != 1) | |
25d0479a | 688 | pr_err("Erroneous page count. Leaking pages.\n"); |
8e7e7052 JG |
689 | __free_page(pages[i]); |
690 | pages[i] = NULL; | |
691 | } | |
692 | } | |
693 | return; | |
694 | } | |
695 | ||
696 | spin_lock_irqsave(&pool->lock, irq_flags); | |
697 | for (i = 0; i < npages; i++) { | |
698 | if (pages[i]) { | |
699 | if (page_count(pages[i]) != 1) | |
25d0479a | 700 | pr_err("Erroneous page count. Leaking pages.\n"); |
8e7e7052 JG |
701 | list_add_tail(&pages[i]->lru, &pool->list); |
702 | pages[i] = NULL; | |
703 | pool->npages++; | |
704 | } | |
705 | } | |
706 | /* Check that we don't go over the pool limit */ | |
707 | npages = 0; | |
708 | if (pool->npages > _manager->options.max_size) { | |
709 | npages = pool->npages - _manager->options.max_size; | |
710 | /* free at least NUM_PAGES_TO_ALLOC number of pages | |
711 | * to reduce calls to set_memory_wb */ | |
712 | if (npages < NUM_PAGES_TO_ALLOC) | |
713 | npages = NUM_PAGES_TO_ALLOC; | |
714 | } | |
715 | spin_unlock_irqrestore(&pool->lock, irq_flags); | |
716 | if (npages) | |
881fdaa5 | 717 | ttm_page_pool_free(pool, npages, false); |
8e7e7052 JG |
718 | } |
719 | ||
1403b1a3 PN |
720 | /* |
721 | * On success pages list will hold count number of correctly | |
722 | * cached pages. | |
723 | */ | |
8e7e7052 JG |
724 | static int ttm_get_pages(struct page **pages, unsigned npages, int flags, |
725 | enum ttm_caching_state cstate) | |
1403b1a3 PN |
726 | { |
727 | struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); | |
822c4d9a | 728 | struct list_head plist; |
1403b1a3 | 729 | struct page *p = NULL; |
0e57a3cc | 730 | gfp_t gfp_flags = GFP_USER; |
822c4d9a | 731 | unsigned count; |
1403b1a3 PN |
732 | int r; |
733 | ||
734 | /* set zero flag for page allocation if required */ | |
735 | if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) | |
736 | gfp_flags |= __GFP_ZERO; | |
737 | ||
738 | /* No pool for cached pages */ | |
739 | if (pool == NULL) { | |
740 | if (flags & TTM_PAGE_FLAG_DMA32) | |
741 | gfp_flags |= GFP_DMA32; | |
742 | else | |
e8613c0e | 743 | gfp_flags |= GFP_HIGHUSER; |
1403b1a3 | 744 | |
822c4d9a | 745 | for (r = 0; r < npages; ++r) { |
d87dfdbf | 746 | p = alloc_page(gfp_flags); |
1403b1a3 PN |
747 | if (!p) { |
748 | ||
25d0479a | 749 | pr_err("Unable to allocate page\n"); |
1403b1a3 PN |
750 | return -ENOMEM; |
751 | } | |
d87dfdbf | 752 | |
822c4d9a | 753 | pages[r] = p; |
1403b1a3 PN |
754 | } |
755 | return 0; | |
756 | } | |
757 | ||
1403b1a3 PN |
758 | /* combine zero flag to pool flags */ |
759 | gfp_flags |= pool->gfp_flags; | |
760 | ||
761 | /* First we take pages from the pool */ | |
822c4d9a JG |
762 | INIT_LIST_HEAD(&plist); |
763 | npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); | |
764 | count = 0; | |
765 | list_for_each_entry(p, &plist, lru) { | |
766 | pages[count++] = p; | |
767 | } | |
1403b1a3 PN |
768 | |
769 | /* clear the pages coming from the pool if requested */ | |
770 | if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { | |
822c4d9a | 771 | list_for_each_entry(p, &plist, lru) { |
ac207ed2 ZY |
772 | if (PageHighMem(p)) |
773 | clear_highpage(p); | |
774 | else | |
775 | clear_page(page_address(p)); | |
1403b1a3 PN |
776 | } |
777 | } | |
778 | ||
779 | /* If pool didn't have enough pages allocate new one. */ | |
822c4d9a | 780 | if (npages > 0) { |
1403b1a3 PN |
781 | /* ttm_alloc_new_pages doesn't reference pool so we can run |
782 | * multiple requests in parallel. | |
783 | **/ | |
822c4d9a JG |
784 | INIT_LIST_HEAD(&plist); |
785 | r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages); | |
786 | list_for_each_entry(p, &plist, lru) { | |
787 | pages[count++] = p; | |
788 | } | |
1403b1a3 PN |
789 | if (r) { |
790 | /* If there is any pages in the list put them back to | |
791 | * the pool. */ | |
25d0479a | 792 | pr_err("Failed to allocate extra pages for large request\n"); |
8e7e7052 | 793 | ttm_put_pages(pages, count, flags, cstate); |
1403b1a3 PN |
794 | return r; |
795 | } | |
796 | } | |
797 | ||
1403b1a3 PN |
798 | return 0; |
799 | } | |
800 | ||
3b9c214a | 801 | static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, |
07458661 | 802 | char *name) |
1403b1a3 PN |
803 | { |
804 | spin_lock_init(&pool->lock); | |
805 | pool->fill_lock = false; | |
806 | INIT_LIST_HEAD(&pool->list); | |
07458661 | 807 | pool->npages = pool->nfrees = 0; |
1403b1a3 | 808 | pool->gfp_flags = flags; |
07458661 | 809 | pool->name = name; |
1403b1a3 PN |
810 | } |
811 | ||
c96af79e | 812 | int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) |
1403b1a3 | 813 | { |
c96af79e | 814 | int ret; |
5870a4d9 FJ |
815 | |
816 | WARN_ON(_manager); | |
1403b1a3 | 817 | |
25d0479a | 818 | pr_info("Initializing pool allocator\n"); |
1403b1a3 | 819 | |
5870a4d9 | 820 | _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); |
1403b1a3 | 821 | |
5870a4d9 | 822 | ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc"); |
1403b1a3 | 823 | |
5870a4d9 | 824 | ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc"); |
1403b1a3 | 825 | |
5870a4d9 FJ |
826 | ttm_page_pool_init_locked(&_manager->wc_pool_dma32, |
827 | GFP_USER | GFP_DMA32, "wc dma"); | |
1403b1a3 | 828 | |
5870a4d9 FJ |
829 | ttm_page_pool_init_locked(&_manager->uc_pool_dma32, |
830 | GFP_USER | GFP_DMA32, "uc dma"); | |
1403b1a3 | 831 | |
5870a4d9 FJ |
832 | _manager->options.max_size = max_pages; |
833 | _manager->options.small = SMALL_ALLOCATION; | |
834 | _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; | |
835 | ||
836 | ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, | |
837 | &glob->kobj, "pool"); | |
c96af79e | 838 | if (unlikely(ret != 0)) { |
5870a4d9 FJ |
839 | kobject_put(&_manager->kobj); |
840 | _manager = NULL; | |
c96af79e PN |
841 | return ret; |
842 | } | |
843 | ||
5870a4d9 | 844 | ttm_pool_mm_shrink_init(_manager); |
1403b1a3 PN |
845 | |
846 | return 0; | |
847 | } | |
848 | ||
0e57a3cc | 849 | void ttm_page_alloc_fini(void) |
1403b1a3 PN |
850 | { |
851 | int i; | |
852 | ||
25d0479a | 853 | pr_info("Finalizing pool allocator\n"); |
5870a4d9 | 854 | ttm_pool_mm_shrink_fini(_manager); |
1403b1a3 | 855 | |
881fdaa5 | 856 | /* OK to use static buffer since global mutex is no longer used. */ |
1403b1a3 | 857 | for (i = 0; i < NUM_POOLS; ++i) |
881fdaa5 | 858 | ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES, true); |
c96af79e | 859 | |
5870a4d9 FJ |
860 | kobject_put(&_manager->kobj); |
861 | _manager = NULL; | |
1403b1a3 | 862 | } |
07458661 | 863 | |
b1e5f172 JG |
864 | int ttm_pool_populate(struct ttm_tt *ttm) |
865 | { | |
866 | struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; | |
867 | unsigned i; | |
868 | int ret; | |
869 | ||
870 | if (ttm->state != tt_unpopulated) | |
871 | return 0; | |
872 | ||
873 | for (i = 0; i < ttm->num_pages; ++i) { | |
8e7e7052 JG |
874 | ret = ttm_get_pages(&ttm->pages[i], 1, |
875 | ttm->page_flags, | |
876 | ttm->caching_state); | |
b1e5f172 JG |
877 | if (ret != 0) { |
878 | ttm_pool_unpopulate(ttm); | |
879 | return -ENOMEM; | |
880 | } | |
881 | ||
882 | ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], | |
883 | false, false); | |
884 | if (unlikely(ret != 0)) { | |
885 | ttm_pool_unpopulate(ttm); | |
886 | return -ENOMEM; | |
887 | } | |
888 | } | |
889 | ||
890 | if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { | |
891 | ret = ttm_tt_swapin(ttm); | |
892 | if (unlikely(ret != 0)) { | |
893 | ttm_pool_unpopulate(ttm); | |
894 | return ret; | |
895 | } | |
896 | } | |
897 | ||
898 | ttm->state = tt_unbound; | |
899 | return 0; | |
900 | } | |
901 | EXPORT_SYMBOL(ttm_pool_populate); | |
902 | ||
903 | void ttm_pool_unpopulate(struct ttm_tt *ttm) | |
904 | { | |
905 | unsigned i; | |
906 | ||
907 | for (i = 0; i < ttm->num_pages; ++i) { | |
908 | if (ttm->pages[i]) { | |
909 | ttm_mem_global_free_page(ttm->glob->mem_glob, | |
910 | ttm->pages[i]); | |
911 | ttm_put_pages(&ttm->pages[i], 1, | |
912 | ttm->page_flags, | |
8e7e7052 | 913 | ttm->caching_state); |
b1e5f172 JG |
914 | } |
915 | } | |
916 | ttm->state = tt_unpopulated; | |
917 | } | |
918 | EXPORT_SYMBOL(ttm_pool_unpopulate); | |
919 | ||
07458661 PN |
920 | int ttm_page_alloc_debugfs(struct seq_file *m, void *data) |
921 | { | |
922 | struct ttm_page_pool *p; | |
923 | unsigned i; | |
924 | char *h[] = {"pool", "refills", "pages freed", "size"}; | |
5870a4d9 | 925 | if (!_manager) { |
07458661 PN |
926 | seq_printf(m, "No pool allocator running.\n"); |
927 | return 0; | |
928 | } | |
929 | seq_printf(m, "%6s %12s %13s %8s\n", | |
930 | h[0], h[1], h[2], h[3]); | |
931 | for (i = 0; i < NUM_POOLS; ++i) { | |
5870a4d9 | 932 | p = &_manager->pools[i]; |
07458661 PN |
933 | |
934 | seq_printf(m, "%6s %12ld %13ld %8d\n", | |
935 | p->name, p->nrefills, | |
936 | p->nfrees, p->npages); | |
937 | } | |
938 | return 0; | |
939 | } | |
940 | EXPORT_SYMBOL(ttm_page_alloc_debugfs); |