projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915/debug: Remove default WATCH_BUF
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
5a0e3ad6 34#include <linux/slab.h>
673a394b 35#include <linux/swap.h>
79e53945 36#include <linux/pci.h>
f8f235e5 37#include <linux/intel-gtt.h>
673a394b 38
0108a3ed 39static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
ba3d8d74
DV		 40
41static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
42 bool pipelined);
e47c68e9
EA		 43static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
44static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
e47c68e9
EA		 45static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
46 int write);
47static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
48 uint64_t offset,
49 uint64_t size);
50static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
2cf34d7b
CW		 51static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
52 bool interruptible);
de151cf6
JB		 53static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
54 unsigned alignment);
de151cf6 55static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
71acb5eb
DA		 56static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
57 struct drm_i915_gem_pwrite *args,
58 struct drm_file *file_priv);
be72615b 59static void i915_gem_free_object_tail(struct drm_gem_object *obj);
673a394b 60
31169714
CW		 61static LIST_HEAD(shrink_list);
62static DEFINE_SPINLOCK(shrink_list_lock);
63
30dbf0c0
CW		 64int
65i915_gem_check_is_wedged(struct drm_device *dev)
66{
67 struct drm_i915_private *dev_priv = dev->dev_private;
68 struct completion *x = &dev_priv->error_completion;
69 unsigned long flags;
70 int ret;
71
72 if (!atomic_read(&dev_priv->mm.wedged))
73 return 0;
74
75 ret = wait_for_completion_interruptible(x);
76 if (ret)
77 return ret;
78
79 /* Success, we reset the GPU! */
80 if (!atomic_read(&dev_priv->mm.wedged))
81 return 0;
82
83 /* GPU is hung, bump the completion count to account for
84 * the token we just consumed so that we never hit zero and
85 * end up waiting upon a subsequent completion event that
86 * will never happen.
87 */
88 spin_lock_irqsave(&x->wait.lock, flags);
89 x->done++;
90 spin_unlock_irqrestore(&x->wait.lock, flags);
91 return -EIO;
92}
93
76c1dec1
CW		 94static int i915_mutex_lock_interruptible(struct drm_device *dev)
95{
96 struct drm_i915_private *dev_priv = dev->dev_private;
97 int ret;
98
99 ret = i915_gem_check_is_wedged(dev);
100 if (ret)
101 return ret;
102
103 ret = mutex_lock_interruptible(&dev->struct_mutex);
104 if (ret)
105 return ret;
106
107 if (atomic_read(&dev_priv->mm.wedged)) {
108 mutex_unlock(&dev->struct_mutex);
109 return -EAGAIN;
110 }
111
112 return 0;
113}
30dbf0c0 114
7d1c4804
CW		 115static inline bool
116i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
117{
118 return obj_priv->gtt_space &&
119 !obj_priv->active &&
120 obj_priv->pin_count == 0;
121}
122
79e53945
JB		 123int i915_gem_do_init(struct drm_device *dev, unsigned long start,
124 unsigned long end)
673a394b
EA		 125{
126 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 127
79e53945
JB		 128 if (start >= end ||
129 (start & (PAGE_SIZE - 1)) != 0 ||
130 (end & (PAGE_SIZE - 1)) != 0) {
673a394b
EA		 131 return -EINVAL;
132 }
133
79e53945
JB		 134 drm_mm_init(&dev_priv->mm.gtt_space, start,
135 end - start);
673a394b 136
79e53945
JB		 137 dev->gtt_total = (uint32_t) (end - start);
138
139 return 0;
140}
673a394b 141
79e53945
JB		 142int
143i915_gem_init_ioctl(struct drm_device *dev, void *data,
144 struct drm_file *file_priv)
145{
146 struct drm_i915_gem_init *args = data;
147 int ret;
148
149 mutex_lock(&dev->struct_mutex);
150 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
673a394b
EA		 151 mutex_unlock(&dev->struct_mutex);
152
79e53945 153 return ret;
673a394b
EA		 154}
155
5a125c3c
EA		 156int
157i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
158 struct drm_file *file_priv)
159{
5a125c3c 160 struct drm_i915_gem_get_aperture *args = data;
5a125c3c
EA		 161
162 if (!(dev->driver->driver_features & DRIVER_GEM))
163 return -ENODEV;
164
165 args->aper_size = dev->gtt_total;
2678d9d6
KP		 166 args->aper_available_size = (args->aper_size -
167 atomic_read(&dev->pin_memory));
5a125c3c
EA		 168
169 return 0;
170}
171
673a394b
EA		 172
173/**
174 * Creates a new mm object and returns a handle to it.
175 */
176int
177i915_gem_create_ioctl(struct drm_device *dev, void *data,
178 struct drm_file *file_priv)
179{
180 struct drm_i915_gem_create *args = data;
181 struct drm_gem_object *obj;
a1a2d1d3
PP		 182 int ret;
183 u32 handle;
673a394b
EA		 184
185 args->size = roundup(args->size, PAGE_SIZE);
186
187 /* Allocate the new object */
ac52bc56 188 obj = i915_gem_alloc_object(dev, args->size);
673a394b
EA		 189 if (obj == NULL)
190 return -ENOMEM;
191
192 ret = drm_gem_handle_create(file_priv, obj, &handle);
1dfd9754
CW		 193 if (ret) {
194 drm_gem_object_unreference_unlocked(obj);
673a394b 195 return ret;
1dfd9754 196 }
673a394b 197
1dfd9754
CW		 198 /* Sink the floating reference from kref_init(handlecount) */
199 drm_gem_object_handle_unreference_unlocked(obj);
673a394b 200
1dfd9754 201 args->handle = handle;
673a394b
EA		 202 return 0;
203}
204
eb01459f
EA		 205static inline int
206fast_shmem_read(struct page **pages,
207 loff_t page_base, int page_offset,
208 char __user *data,
209 int length)
210{
211 char __iomem *vaddr;
2bc43b5c 212 int unwritten;
eb01459f
EA		 213
214 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
215 if (vaddr == NULL)
216 return -ENOMEM;
2bc43b5c 217 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
eb01459f
EA		 218 kunmap_atomic(vaddr, KM_USER0);
219
2bc43b5c
FM		 220 if (unwritten)
221 return -EFAULT;
222
223 return 0;
eb01459f
EA		 224}
225
280b713b
EA		 226static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
227{
228 drm_i915_private_t *dev_priv = obj->dev->dev_private;
23010e43 229 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
280b713b
EA		 230
231 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
232 obj_priv->tiling_mode != I915_TILING_NONE;
233}
234
99a03df5 235static inline void
40123c1f
EA		 236slow_shmem_copy(struct page *dst_page,
237 int dst_offset,
238 struct page *src_page,
239 int src_offset,
240 int length)
241{
242 char *dst_vaddr, *src_vaddr;
243
99a03df5
CW		 244 dst_vaddr = kmap(dst_page);
245 src_vaddr = kmap(src_page);
40123c1f
EA		 246
247 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
248
99a03df5
CW		 249 kunmap(src_page);
250 kunmap(dst_page);
40123c1f
EA		 251}
252
99a03df5 253static inline void
280b713b
EA		 254slow_shmem_bit17_copy(struct page *gpu_page,
255 int gpu_offset,
256 struct page *cpu_page,
257 int cpu_offset,
258 int length,
259 int is_read)
260{
261 char *gpu_vaddr, *cpu_vaddr;
262
263 /* Use the unswizzled path if this page isn't affected. */
264 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
265 if (is_read)
266 return slow_shmem_copy(cpu_page, cpu_offset,
267 gpu_page, gpu_offset, length);
268 else
269 return slow_shmem_copy(gpu_page, gpu_offset,
270 cpu_page, cpu_offset, length);
271 }
272
99a03df5
CW		 273 gpu_vaddr = kmap(gpu_page);
274 cpu_vaddr = kmap(cpu_page);
280b713b
EA		 275
276 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
277 * XORing with the other bits (A9 for Y, A9 and A10 for X)
278 */
279 while (length > 0) {
280 int cacheline_end = ALIGN(gpu_offset + 1, 64);
281 int this_length = min(cacheline_end - gpu_offset, length);
282 int swizzled_gpu_offset = gpu_offset ^ 64;
283
284 if (is_read) {
285 memcpy(cpu_vaddr + cpu_offset,
286 gpu_vaddr + swizzled_gpu_offset,
287 this_length);
288 } else {
289 memcpy(gpu_vaddr + swizzled_gpu_offset,
290 cpu_vaddr + cpu_offset,
291 this_length);
292 }
293 cpu_offset += this_length;
294 gpu_offset += this_length;
295 length -= this_length;
296 }
297
99a03df5
CW		 298 kunmap(cpu_page);
299 kunmap(gpu_page);
280b713b
EA		 300}
301
eb01459f
EA		 302/**
303 * This is the fast shmem pread path, which attempts to copy_from_user directly
304 * from the backing pages of the object to the user's address space. On a
305 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
306 */
307static int
308i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
309 struct drm_i915_gem_pread *args,
310 struct drm_file *file_priv)
311{
23010e43 312 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
eb01459f
EA		 313 ssize_t remain;
314 loff_t offset, page_base;
315 char __user *user_data;
316 int page_offset, page_length;
317 int ret;
318
319 user_data = (char __user *) (uintptr_t) args->data_ptr;
320 remain = args->size;
321
76c1dec1
CW		 322 ret = i915_mutex_lock_interruptible(dev);
323 if (ret)
324 return ret;
eb01459f 325
4bdadb97 326 ret = i915_gem_object_get_pages(obj, 0);
eb01459f
EA		 327 if (ret != 0)
328 goto fail_unlock;
329
330 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
331 args->size);
332 if (ret != 0)
333 goto fail_put_pages;
334
23010e43 335 obj_priv = to_intel_bo(obj);
eb01459f
EA		 336 offset = args->offset;
337
338 while (remain > 0) {
339 /* Operation in this page
340 *
341 * page_base = page offset within aperture
342 * page_offset = offset within page
343 * page_length = bytes to copy for this page
344 */
345 page_base = (offset & ~(PAGE_SIZE-1));
346 page_offset = offset & (PAGE_SIZE-1);
347 page_length = remain;
348 if ((page_offset + remain) > PAGE_SIZE)
349 page_length = PAGE_SIZE - page_offset;
350
351 ret = fast_shmem_read(obj_priv->pages,
352 page_base, page_offset,
353 user_data, page_length);
354 if (ret)
355 goto fail_put_pages;
356
357 remain -= page_length;
358 user_data += page_length;
359 offset += page_length;
360 }
361
362fail_put_pages:
363 i915_gem_object_put_pages(obj);
364fail_unlock:
365 mutex_unlock(&dev->struct_mutex);
366
367 return ret;
368}
369
07f73f69
CW		 370static int
371i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
372{
373 int ret;
374
4bdadb97 375 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
07f73f69
CW		 376
377 /* If we've insufficient memory to map in the pages, attempt
378 * to make some space by throwing out some old buffers.
379 */
380 if (ret == -ENOMEM) {
381 struct drm_device *dev = obj->dev;
07f73f69 382
0108a3ed
DV		 383 ret = i915_gem_evict_something(dev, obj->size,
384 i915_gem_get_gtt_alignment(obj));
07f73f69
CW		 385 if (ret)
386 return ret;
387
4bdadb97 388 ret = i915_gem_object_get_pages(obj, 0);
07f73f69
CW		 389 }
390
391 return ret;
392}
393
eb01459f
EA		 394/**
395 * This is the fallback shmem pread path, which allocates temporary storage
396 * in kernel space to copy_to_user into outside of the struct_mutex, so we
397 * can copy out of the object's backing pages while holding the struct mutex
398 * and not take page faults.
399 */
400static int
401i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
402 struct drm_i915_gem_pread *args,
403 struct drm_file *file_priv)
404{
23010e43 405 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
eb01459f
EA		 406 struct mm_struct *mm = current->mm;
407 struct page **user_pages;
408 ssize_t remain;
409 loff_t offset, pinned_pages, i;
410 loff_t first_data_page, last_data_page, num_pages;
411 int shmem_page_index, shmem_page_offset;
412 int data_page_index, data_page_offset;
413 int page_length;
414 int ret;
415 uint64_t data_ptr = args->data_ptr;
280b713b 416 int do_bit17_swizzling;
eb01459f
EA		 417
418 remain = args->size;
419
420 /* Pin the user pages containing the data. We can't fault while
421 * holding the struct mutex, yet we want to hold it while
422 * dereferencing the user data.
423 */
424 first_data_page = data_ptr / PAGE_SIZE;
425 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
426 num_pages = last_data_page - first_data_page + 1;
427
8e7d2b2c 428 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
eb01459f
EA		 429 if (user_pages == NULL)
430 return -ENOMEM;
431
432 down_read(&mm->mmap_sem);
433 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 434 num_pages, 1, 0, user_pages, NULL);
eb01459f
EA		 435 up_read(&mm->mmap_sem);
436 if (pinned_pages < num_pages) {
437 ret = -EFAULT;
438 goto fail_put_user_pages;
439 }
440
280b713b
EA		 441 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
442
76c1dec1
CW		 443 ret = i915_mutex_lock_interruptible(dev);
444 if (ret)
445 goto fail_put_user_pages;
eb01459f 446
07f73f69
CW		 447 ret = i915_gem_object_get_pages_or_evict(obj);
448 if (ret)
eb01459f
EA		 449 goto fail_unlock;
450
451 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
452 args->size);
453 if (ret != 0)
454 goto fail_put_pages;
455
23010e43 456 obj_priv = to_intel_bo(obj);
eb01459f
EA		 457 offset = args->offset;
458
459 while (remain > 0) {
460 /* Operation in this page
461 *
462 * shmem_page_index = page number within shmem file
463 * shmem_page_offset = offset within page in shmem file
464 * data_page_index = page number in get_user_pages return
465 * data_page_offset = offset with data_page_index page.
466 * page_length = bytes to copy for this page
467 */
468 shmem_page_index = offset / PAGE_SIZE;
469 shmem_page_offset = offset & ~PAGE_MASK;
470 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
471 data_page_offset = data_ptr & ~PAGE_MASK;
472
473 page_length = remain;
474 if ((shmem_page_offset + page_length) > PAGE_SIZE)
475 page_length = PAGE_SIZE - shmem_page_offset;
476 if ((data_page_offset + page_length) > PAGE_SIZE)
477 page_length = PAGE_SIZE - data_page_offset;
478
280b713b 479 if (do_bit17_swizzling) {
99a03df5 480 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
280b713b 481 shmem_page_offset,
99a03df5
CW		 482 user_pages[data_page_index],
483 data_page_offset,
484 page_length,
485 1);
486 } else {
487 slow_shmem_copy(user_pages[data_page_index],
488 data_page_offset,
489 obj_priv->pages[shmem_page_index],
490 shmem_page_offset,
491 page_length);
280b713b 492 }
eb01459f
EA		 493
494 remain -= page_length;
495 data_ptr += page_length;
496 offset += page_length;
497 }
498
499fail_put_pages:
500 i915_gem_object_put_pages(obj);
501fail_unlock:
502 mutex_unlock(&dev->struct_mutex);
503fail_put_user_pages:
504 for (i = 0; i < pinned_pages; i++) {
505 SetPageDirty(user_pages[i]);
506 page_cache_release(user_pages[i]);
507 }
8e7d2b2c 508 drm_free_large(user_pages);
eb01459f
EA		 509
510 return ret;
511}
512
673a394b
EA		 513/**
514 * Reads data from the object referenced by handle.
515 *
516 * On error, the contents of *data are undefined.
517 */
518int
519i915_gem_pread_ioctl(struct drm_device *dev, void *data,
520 struct drm_file *file_priv)
521{
522 struct drm_i915_gem_pread *args = data;
523 struct drm_gem_object *obj;
524 struct drm_i915_gem_object *obj_priv;
673a394b
EA		 525 int ret;
526
527 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
528 if (obj == NULL)
bf79cb91 529 return -ENOENT;
23010e43 530 obj_priv = to_intel_bo(obj);
673a394b
EA		 531
532 /* Bounds check source.
533 *
534 * XXX: This could use review for overflow issues...
535 */
536 if (args->offset > obj->size || args->size > obj->size ||
537 args->offset + args->size > obj->size) {
bc9025bd 538 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 539 return -EINVAL;
540 }
541
280b713b 542 if (i915_gem_object_needs_bit17_swizzle(obj)) {
eb01459f 543 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
280b713b
EA		 544 } else {
545 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
546 if (ret != 0)
547 ret = i915_gem_shmem_pread_slow(dev, obj, args,
548 file_priv);
549 }
673a394b 550
bc9025bd 551 drm_gem_object_unreference_unlocked(obj);
673a394b 552
eb01459f 553 return ret;
673a394b
EA		 554}
555
0839ccb8
KP		 556/* This is the fast write path which cannot handle
557 * page faults in the source data
9b7530cc 558 */
0839ccb8
KP		 559
560static inline int
561fast_user_write(struct io_mapping *mapping,
562 loff_t page_base, int page_offset,
563 char __user *user_data,
564 int length)
9b7530cc 565{
9b7530cc 566 char *vaddr_atomic;
0839ccb8 567 unsigned long unwritten;
9b7530cc 568
fca3ec01 569 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
0839ccb8
KP		 570 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
571 user_data, length);
fca3ec01 572 io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
0839ccb8
KP		 573 if (unwritten)
574 return -EFAULT;
575 return 0;
576}
577
578/* Here's the write path which can sleep for
579 * page faults
580 */
581
ab34c226 582static inline void
3de09aa3
EA		 583slow_kernel_write(struct io_mapping *mapping,
584 loff_t gtt_base, int gtt_offset,
585 struct page *user_page, int user_offset,
586 int length)
0839ccb8 587{
ab34c226
CW		 588 char __iomem *dst_vaddr;
589 char *src_vaddr;
0839ccb8 590
ab34c226
CW		 591 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
592 src_vaddr = kmap(user_page);
593
594 memcpy_toio(dst_vaddr + gtt_offset,
595 src_vaddr + user_offset,
596 length);
597
598 kunmap(user_page);
599 io_mapping_unmap(dst_vaddr);
9b7530cc
LT		 600}
601
40123c1f
EA		 602static inline int
603fast_shmem_write(struct page **pages,
604 loff_t page_base, int page_offset,
605 char __user *data,
606 int length)
607{
608 char __iomem *vaddr;
d0088775 609 unsigned long unwritten;
40123c1f
EA		 610
611 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
612 if (vaddr == NULL)
613 return -ENOMEM;
d0088775 614 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
40123c1f
EA		 615 kunmap_atomic(vaddr, KM_USER0);
616
d0088775
DA		 617 if (unwritten)
618 return -EFAULT;
40123c1f
EA		 619 return 0;
620}
621
3de09aa3
EA		 622/**
623 * This is the fast pwrite path, where we copy the data directly from the
624 * user into the GTT, uncached.
625 */
673a394b 626static int
3de09aa3
EA		 627i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
628 struct drm_i915_gem_pwrite *args,
629 struct drm_file *file_priv)
673a394b 630{
23010e43 631 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
0839ccb8 632 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 633 ssize_t remain;
0839ccb8 634 loff_t offset, page_base;
673a394b 635 char __user *user_data;
0839ccb8
KP		 636 int page_offset, page_length;
637 int ret;
673a394b
EA		 638
639 user_data = (char __user *) (uintptr_t) args->data_ptr;
640 remain = args->size;
641 if (!access_ok(VERIFY_READ, user_data, remain))
642 return -EFAULT;
643
76c1dec1
CW		 644 ret = i915_mutex_lock_interruptible(dev);
645 if (ret)
646 return ret;
673a394b 647
673a394b
EA		 648 ret = i915_gem_object_pin(obj, 0);
649 if (ret) {
650 mutex_unlock(&dev->struct_mutex);
651 return ret;
652 }
2ef7eeaa 653 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
673a394b
EA		 654 if (ret)
655 goto fail;
656
23010e43 657 obj_priv = to_intel_bo(obj);
673a394b 658 offset = obj_priv->gtt_offset + args->offset;
673a394b
EA		 659
660 while (remain > 0) {
661 /* Operation in this page
662 *
0839ccb8
KP		 663 * page_base = page offset within aperture
664 * page_offset = offset within page
665 * page_length = bytes to copy for this page
673a394b 666 */
0839ccb8
KP		 667 page_base = (offset & ~(PAGE_SIZE-1));
668 page_offset = offset & (PAGE_SIZE-1);
669 page_length = remain;
670 if ((page_offset + remain) > PAGE_SIZE)
671 page_length = PAGE_SIZE - page_offset;
672
673 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
674 page_offset, user_data, page_length);
675
676 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 677 * source page isn't available. Return the error and we'll
678 * retry in the slow path.
0839ccb8 679 */
3de09aa3
EA		 680 if (ret)
681 goto fail;
673a394b 682
0839ccb8
KP		 683 remain -= page_length;
684 user_data += page_length;
685 offset += page_length;
673a394b 686 }
673a394b
EA		 687
688fail:
689 i915_gem_object_unpin(obj);
690 mutex_unlock(&dev->struct_mutex);
691
692 return ret;
693}
694
3de09aa3
EA		 695/**
696 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
697 * the memory and maps it using kmap_atomic for copying.
698 *
699 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
700 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
701 */
3043c60c 702static int
3de09aa3
EA		 703i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
704 struct drm_i915_gem_pwrite *args,
705 struct drm_file *file_priv)
673a394b 706{
23010e43 707 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3de09aa3
EA		 708 drm_i915_private_t *dev_priv = dev->dev_private;
709 ssize_t remain;
710 loff_t gtt_page_base, offset;
711 loff_t first_data_page, last_data_page, num_pages;
712 loff_t pinned_pages, i;
713 struct page **user_pages;
714 struct mm_struct *mm = current->mm;
715 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 716 int ret;
3de09aa3
EA		 717 uint64_t data_ptr = args->data_ptr;
718
719 remain = args->size;
720
721 /* Pin the user pages containing the data. We can't fault while
722 * holding the struct mutex, and all of the pwrite implementations
723 * want to hold it while dereferencing the user data.
724 */
725 first_data_page = data_ptr / PAGE_SIZE;
726 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
727 num_pages = last_data_page - first_data_page + 1;
728
8e7d2b2c 729 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
3de09aa3
EA		 730 if (user_pages == NULL)
731 return -ENOMEM;
732
733 down_read(&mm->mmap_sem);
734 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
735 num_pages, 0, 0, user_pages, NULL);
736 up_read(&mm->mmap_sem);
737 if (pinned_pages < num_pages) {
738 ret = -EFAULT;
739 goto out_unpin_pages;
740 }
673a394b 741
76c1dec1
CW		 742 ret = i915_mutex_lock_interruptible(dev);
743 if (ret)
744 goto out_unpin_pages;
745
3de09aa3
EA		 746 ret = i915_gem_object_pin(obj, 0);
747 if (ret)
748 goto out_unlock;
749
750 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
751 if (ret)
752 goto out_unpin_object;
753
23010e43 754 obj_priv = to_intel_bo(obj);
3de09aa3
EA		 755 offset = obj_priv->gtt_offset + args->offset;
756
757 while (remain > 0) {
758 /* Operation in this page
759 *
760 * gtt_page_base = page offset within aperture
761 * gtt_page_offset = offset within page in aperture
762 * data_page_index = page number in get_user_pages return
763 * data_page_offset = offset with data_page_index page.
764 * page_length = bytes to copy for this page
765 */
766 gtt_page_base = offset & PAGE_MASK;
767 gtt_page_offset = offset & ~PAGE_MASK;
768 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
769 data_page_offset = data_ptr & ~PAGE_MASK;
770
771 page_length = remain;
772 if ((gtt_page_offset + page_length) > PAGE_SIZE)
773 page_length = PAGE_SIZE - gtt_page_offset;
774 if ((data_page_offset + page_length) > PAGE_SIZE)
775 page_length = PAGE_SIZE - data_page_offset;
776
ab34c226
CW		 777 slow_kernel_write(dev_priv->mm.gtt_mapping,
778 gtt_page_base, gtt_page_offset,
779 user_pages[data_page_index],
780 data_page_offset,
781 page_length);
3de09aa3
EA		 782
783 remain -= page_length;
784 offset += page_length;
785 data_ptr += page_length;
786 }
787
788out_unpin_object:
789 i915_gem_object_unpin(obj);
790out_unlock:
791 mutex_unlock(&dev->struct_mutex);
792out_unpin_pages:
793 for (i = 0; i < pinned_pages; i++)
794 page_cache_release(user_pages[i]);
8e7d2b2c 795 drm_free_large(user_pages);
3de09aa3
EA		 796
797 return ret;
798}
799
40123c1f
EA		 800/**
801 * This is the fast shmem pwrite path, which attempts to directly
802 * copy_from_user into the kmapped pages backing the object.
803 */
3043c60c 804static int
40123c1f
EA		 805i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
806 struct drm_i915_gem_pwrite *args,
807 struct drm_file *file_priv)
673a394b 808{
23010e43 809 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
40123c1f
EA		 810 ssize_t remain;
811 loff_t offset, page_base;
812 char __user *user_data;
813 int page_offset, page_length;
673a394b 814 int ret;
40123c1f
EA		 815
816 user_data = (char __user *) (uintptr_t) args->data_ptr;
817 remain = args->size;
673a394b 818
76c1dec1
CW		 819 ret = i915_mutex_lock_interruptible(dev);
820 if (ret)
821 return ret;
673a394b 822
4bdadb97 823 ret = i915_gem_object_get_pages(obj, 0);
40123c1f
EA		 824 if (ret != 0)
825 goto fail_unlock;
673a394b 826
e47c68e9 827 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
40123c1f
EA		 828 if (ret != 0)
829 goto fail_put_pages;
830
23010e43 831 obj_priv = to_intel_bo(obj);
40123c1f
EA		 832 offset = args->offset;
833 obj_priv->dirty = 1;
834
835 while (remain > 0) {
836 /* Operation in this page
837 *
838 * page_base = page offset within aperture
839 * page_offset = offset within page
840 * page_length = bytes to copy for this page
841 */
842 page_base = (offset & ~(PAGE_SIZE-1));
843 page_offset = offset & (PAGE_SIZE-1);
844 page_length = remain;
845 if ((page_offset + remain) > PAGE_SIZE)
846 page_length = PAGE_SIZE - page_offset;
847
848 ret = fast_shmem_write(obj_priv->pages,
849 page_base, page_offset,
850 user_data, page_length);
851 if (ret)
852 goto fail_put_pages;
853
854 remain -= page_length;
855 user_data += page_length;
856 offset += page_length;
857 }
858
859fail_put_pages:
860 i915_gem_object_put_pages(obj);
861fail_unlock:
862 mutex_unlock(&dev->struct_mutex);
863
864 return ret;
865}
866
867/**
868 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
869 * the memory and maps it using kmap_atomic for copying.
870 *
871 * This avoids taking mmap_sem for faulting on the user's address while the
872 * struct_mutex is held.
873 */
874static int
875i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
876 struct drm_i915_gem_pwrite *args,
877 struct drm_file *file_priv)
878{
23010e43 879 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
40123c1f
EA		 880 struct mm_struct *mm = current->mm;
881 struct page **user_pages;
882 ssize_t remain;
883 loff_t offset, pinned_pages, i;
884 loff_t first_data_page, last_data_page, num_pages;
885 int shmem_page_index, shmem_page_offset;
886 int data_page_index, data_page_offset;
887 int page_length;
888 int ret;
889 uint64_t data_ptr = args->data_ptr;
280b713b 890 int do_bit17_swizzling;
40123c1f
EA		 891
892 remain = args->size;
893
894 /* Pin the user pages containing the data. We can't fault while
895 * holding the struct mutex, and all of the pwrite implementations
896 * want to hold it while dereferencing the user data.
897 */
898 first_data_page = data_ptr / PAGE_SIZE;
899 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
900 num_pages = last_data_page - first_data_page + 1;
901
8e7d2b2c 902 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
40123c1f
EA		 903 if (user_pages == NULL)
904 return -ENOMEM;
905
906 down_read(&mm->mmap_sem);
907 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
908 num_pages, 0, 0, user_pages, NULL);
909 up_read(&mm->mmap_sem);
910 if (pinned_pages < num_pages) {
911 ret = -EFAULT;
912 goto fail_put_user_pages;
673a394b
EA		 913 }
914
280b713b
EA		 915 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
916
76c1dec1
CW		 917 ret = i915_mutex_lock_interruptible(dev);
918 if (ret)
919 goto fail_put_user_pages;
40123c1f 920
07f73f69
CW		 921 ret = i915_gem_object_get_pages_or_evict(obj);
922 if (ret)
40123c1f
EA		 923 goto fail_unlock;
924
925 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
926 if (ret != 0)
927 goto fail_put_pages;
928
23010e43 929 obj_priv = to_intel_bo(obj);
673a394b 930 offset = args->offset;
40123c1f 931 obj_priv->dirty = 1;
673a394b 932
40123c1f
EA		 933 while (remain > 0) {
934 /* Operation in this page
935 *
936 * shmem_page_index = page number within shmem file
937 * shmem_page_offset = offset within page in shmem file
938 * data_page_index = page number in get_user_pages return
939 * data_page_offset = offset with data_page_index page.
940 * page_length = bytes to copy for this page
941 */
942 shmem_page_index = offset / PAGE_SIZE;
943 shmem_page_offset = offset & ~PAGE_MASK;
944 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
945 data_page_offset = data_ptr & ~PAGE_MASK;
946
947 page_length = remain;
948 if ((shmem_page_offset + page_length) > PAGE_SIZE)
949 page_length = PAGE_SIZE - shmem_page_offset;
950 if ((data_page_offset + page_length) > PAGE_SIZE)
951 page_length = PAGE_SIZE - data_page_offset;
952
280b713b 953 if (do_bit17_swizzling) {
99a03df5 954 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
280b713b
EA		 955 shmem_page_offset,
956 user_pages[data_page_index],
957 data_page_offset,
99a03df5
CW		 958 page_length,
959 0);
960 } else {
961 slow_shmem_copy(obj_priv->pages[shmem_page_index],
962 shmem_page_offset,
963 user_pages[data_page_index],
964 data_page_offset,
965 page_length);
280b713b 966 }
40123c1f
EA		 967
968 remain -= page_length;
969 data_ptr += page_length;
970 offset += page_length;
673a394b
EA		 971 }
972
40123c1f
EA		 973fail_put_pages:
974 i915_gem_object_put_pages(obj);
975fail_unlock:
673a394b 976 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 977fail_put_user_pages:
978 for (i = 0; i < pinned_pages; i++)
979 page_cache_release(user_pages[i]);
8e7d2b2c 980 drm_free_large(user_pages);
673a394b 981
40123c1f 982 return ret;
673a394b
EA		 983}
984
985/**
986 * Writes data to the object referenced by handle.
987 *
988 * On error, the contents of the buffer that were to be modified are undefined.
989 */
990int
991i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
992 struct drm_file *file_priv)
993{
994 struct drm_i915_gem_pwrite *args = data;
995 struct drm_gem_object *obj;
996 struct drm_i915_gem_object *obj_priv;
997 int ret = 0;
998
999 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1000 if (obj == NULL)
bf79cb91 1001 return -ENOENT;
23010e43 1002 obj_priv = to_intel_bo(obj);
673a394b
EA		 1003
1004 /* Bounds check destination.
1005 *
1006 * XXX: This could use review for overflow issues...
1007 */
1008 if (args->offset > obj->size || args->size > obj->size ||
1009 args->offset + args->size > obj->size) {
bc9025bd 1010 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1011 return -EINVAL;
1012 }
1013
1014 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1015 * it would end up going through the fenced access, and we'll get
1016 * different detiling behavior between reading and writing.
1017 * pread/pwrite currently are reading and writing from the CPU
1018 * perspective, requiring manual detiling by the client.
1019 */
71acb5eb
DA		 1020 if (obj_priv->phys_obj)
1021 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
1022 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
9b8c4a0b
CW		 1023 dev->gtt_total != 0 &&
1024 obj->write_domain != I915_GEM_DOMAIN_CPU) {
3de09aa3
EA		 1025 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
1026 if (ret == -EFAULT) {
1027 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
1028 file_priv);
1029 }
280b713b
EA		 1030 } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
1031 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
40123c1f
EA		 1032 } else {
1033 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
1034 if (ret == -EFAULT) {
1035 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
1036 file_priv);
1037 }
1038 }
673a394b
EA		 1039
1040#if WATCH_PWRITE
1041 if (ret)
1042 DRM_INFO("pwrite failed %d\n", ret);
1043#endif
1044
bc9025bd 1045 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1046
1047 return ret;
1048}
1049
1050/**
2ef7eeaa
EA		 1051 * Called when user space prepares to use an object with the CPU, either
1052 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 1053 */
1054int
1055i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1056 struct drm_file *file_priv)
1057{
a09ba7fa 1058 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 1059 struct drm_i915_gem_set_domain *args = data;
1060 struct drm_gem_object *obj;
652c393a 1061 struct drm_i915_gem_object *obj_priv;
2ef7eeaa
EA		 1062 uint32_t read_domains = args->read_domains;
1063 uint32_t write_domain = args->write_domain;
673a394b
EA		 1064 int ret;
1065
1066 if (!(dev->driver->driver_features & DRIVER_GEM))
1067 return -ENODEV;
1068
2ef7eeaa 1069 /* Only handle setting domains to types used by the CPU. */
21d509e3 1070 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1071 return -EINVAL;
1072
21d509e3 1073 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1074 return -EINVAL;
1075
1076 /* Having something in the write domain implies it's in the read
1077 * domain, and only that read domain. Enforce that in the request.
1078 */
1079 if (write_domain != 0 && read_domains != write_domain)
1080 return -EINVAL;
1081
673a394b
EA		 1082 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1083 if (obj == NULL)
bf79cb91 1084 return -ENOENT;
23010e43 1085 obj_priv = to_intel_bo(obj);
673a394b 1086
76c1dec1
CW		 1087 ret = i915_mutex_lock_interruptible(dev);
1088 if (ret) {
1089 drm_gem_object_unreference_unlocked(obj);
1090 return ret;
1091 }
652c393a
JB		 1092
1093 intel_mark_busy(dev, obj);
1094
2ef7eeaa
EA		 1095 if (read_domains & I915_GEM_DOMAIN_GTT) {
1096 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392 1097
a09ba7fa
EA		 1098 /* Update the LRU on the fence for the CPU access that's
1099 * about to occur.
1100 */
1101 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
007cc8ac
DV		 1102 struct drm_i915_fence_reg *reg =
1103 &dev_priv->fence_regs[obj_priv->fence_reg];
1104 list_move_tail(&reg->lru_list,
a09ba7fa
EA		 1105 &dev_priv->mm.fence_list);
1106 }
1107
02354392
EA		 1108 /* Silently promote "you're not bound, there was nothing to do"
1109 * to success, since the client was just asking us to
1110 * make sure everything was done.
1111 */
1112 if (ret == -EINVAL)
1113 ret = 0;
2ef7eeaa 1114 } else {
e47c68e9 1115 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1116 }
1117
7d1c4804
CW		 1118 /* Maintain LRU order of "inactive" objects */
1119 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1120 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1121
673a394b
EA		 1122 drm_gem_object_unreference(obj);
1123 mutex_unlock(&dev->struct_mutex);
1124 return ret;
1125}
1126
1127/**
1128 * Called when user space has done writes to this buffer
1129 */
1130int
1131i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1132 struct drm_file *file_priv)
1133{
1134 struct drm_i915_gem_sw_finish *args = data;
1135 struct drm_gem_object *obj;
673a394b
EA		 1136 int ret = 0;
1137
1138 if (!(dev->driver->driver_features & DRIVER_GEM))
1139 return -ENODEV;
1140
673a394b 1141 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
76c1dec1 1142 if (obj == NULL)
bf79cb91 1143 return -ENOENT;
76c1dec1
CW		 1144
1145 ret = i915_mutex_lock_interruptible(dev);
1146 if (ret) {
1147 drm_gem_object_unreference_unlocked(obj);
1148 return ret;
673a394b
EA		 1149 }
1150
673a394b 1151 /* Pinned buffers may be scanout, so flush the cache */
3d2a812a 1152 if (to_intel_bo(obj)->pin_count)
e47c68e9
EA		 1153 i915_gem_object_flush_cpu_write_domain(obj);
1154
673a394b
EA		 1155 drm_gem_object_unreference(obj);
1156 mutex_unlock(&dev->struct_mutex);
1157 return ret;
1158}
1159
1160/**
1161 * Maps the contents of an object, returning the address it is mapped
1162 * into.
1163 *
1164 * While the mapping holds a reference on the contents of the object, it doesn't
1165 * imply a ref on the object itself.
1166 */
1167int
1168i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1169 struct drm_file *file_priv)
1170{
1171 struct drm_i915_gem_mmap *args = data;
1172 struct drm_gem_object *obj;
1173 loff_t offset;
1174 unsigned long addr;
1175
1176 if (!(dev->driver->driver_features & DRIVER_GEM))
1177 return -ENODEV;
1178
1179 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1180 if (obj == NULL)
bf79cb91 1181 return -ENOENT;
673a394b
EA		 1182
1183 offset = args->offset;
1184
1185 down_write(&current->mm->mmap_sem);
1186 addr = do_mmap(obj->filp, 0, args->size,
1187 PROT_READ | PROT_WRITE, MAP_SHARED,
1188 args->offset);
1189 up_write(&current->mm->mmap_sem);
bc9025bd 1190 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1191 if (IS_ERR((void *)addr))
1192 return addr;
1193
1194 args->addr_ptr = (uint64_t) addr;
1195
1196 return 0;
1197}
1198
de151cf6
JB		 1199/**
1200 * i915_gem_fault - fault a page into the GTT
1201 * vma: VMA in question
1202 * vmf: fault info
1203 *
1204 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1205 * from userspace. The fault handler takes care of binding the object to
1206 * the GTT (if needed), allocating and programming a fence register (again,
1207 * only if needed based on whether the old reg is still valid or the object
1208 * is tiled) and inserting a new PTE into the faulting process.
1209 *
1210 * Note that the faulting process may involve evicting existing objects
1211 * from the GTT and/or fence registers to make room. So performance may
1212 * suffer if the GTT working set is large or there are few fence registers
1213 * left.
1214 */
1215int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1216{
1217 struct drm_gem_object *obj = vma->vm_private_data;
1218 struct drm_device *dev = obj->dev;
7d1c4804 1219 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1220 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 1221 pgoff_t page_offset;
1222 unsigned long pfn;
1223 int ret = 0;
0f973f27 1224 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1225
1226 /* We don't use vmf->pgoff since that has the fake offset */
1227 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1228 PAGE_SHIFT;
1229
1230 /* Now bind it into the GTT if needed */
1231 mutex_lock(&dev->struct_mutex);
1232 if (!obj_priv->gtt_space) {
e67b8ce1 1233 ret = i915_gem_object_bind_to_gtt(obj, 0);
c715089f
CW		 1234 if (ret)
1235 goto unlock;
07f4f3e8 1236
07f4f3e8 1237 ret = i915_gem_object_set_to_gtt_domain(obj, write);
c715089f
CW		 1238 if (ret)
1239 goto unlock;
de151cf6
JB		 1240 }
1241
1242 /* Need a new fence register? */
a09ba7fa 1243 if (obj_priv->tiling_mode != I915_TILING_NONE) {
2cf34d7b 1244 ret = i915_gem_object_get_fence_reg(obj, true);
c715089f
CW		 1245 if (ret)
1246 goto unlock;
d9ddcb96 1247 }
de151cf6 1248
7d1c4804
CW		 1249 if (i915_gem_object_is_inactive(obj_priv))
1250 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1251
de151cf6
JB		 1252 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1253 page_offset;
1254
1255 /* Finally, remap it using the new GTT offset */
1256 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1257unlock:
de151cf6
JB		 1258 mutex_unlock(&dev->struct_mutex);
1259
1260 switch (ret) {
c715089f
CW		 1261 case 0:
1262 case -ERESTARTSYS:
1263 return VM_FAULT_NOPAGE;
de151cf6
JB		 1264 case -ENOMEM:
1265 case -EAGAIN:
1266 return VM_FAULT_OOM;
de151cf6 1267 default:
c715089f 1268 return VM_FAULT_SIGBUS;
de151cf6
JB		 1269 }
1270}
1271
1272/**
1273 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1274 * @obj: obj in question
1275 *
1276 * GEM memory mapping works by handing back to userspace a fake mmap offset
1277 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1278 * up the object based on the offset and sets up the various memory mapping
1279 * structures.
1280 *
1281 * This routine allocates and attaches a fake offset for @obj.
1282 */
1283static int
1284i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1285{
1286 struct drm_device *dev = obj->dev;
1287 struct drm_gem_mm *mm = dev->mm_private;
23010e43 1288 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 1289 struct drm_map_list *list;
f77d390c 1290 struct drm_local_map *map;
de151cf6
JB		 1291 int ret = 0;
1292
1293 /* Set the object up for mmap'ing */
1294 list = &obj->map_list;
9a298b2a 1295 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
de151cf6
JB		 1296 if (!list->map)
1297 return -ENOMEM;
1298
1299 map = list->map;
1300 map->type = _DRM_GEM;
1301 map->size = obj->size;
1302 map->handle = obj;
1303
1304 /* Get a DRM GEM mmap offset allocated... */
1305 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1306 obj->size / PAGE_SIZE, 0, 0);
1307 if (!list->file_offset_node) {
1308 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
9e0ae534 1309 ret = -ENOSPC;
de151cf6
JB		 1310 goto out_free_list;
1311 }
1312
1313 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1314 obj->size / PAGE_SIZE, 0);
1315 if (!list->file_offset_node) {
1316 ret = -ENOMEM;
1317 goto out_free_list;
1318 }
1319
1320 list->hash.key = list->file_offset_node->start;
9e0ae534
CW		 1321 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1322 if (ret) {
de151cf6
JB		 1323 DRM_ERROR("failed to add to map hash\n");
1324 goto out_free_mm;
1325 }
1326
1327 /* By now we should be all set, any drm_mmap request on the offset
1328 * below will get to our mmap & fault handler */
1329 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1330
1331 return 0;
1332
1333out_free_mm:
1334 drm_mm_put_block(list->file_offset_node);
1335out_free_list:
9a298b2a 1336 kfree(list->map);
de151cf6
JB		 1337
1338 return ret;
1339}
1340
901782b2
CW		 1341/**
1342 * i915_gem_release_mmap - remove physical page mappings
1343 * @obj: obj in question
1344 *
af901ca1 1345 * Preserve the reservation of the mmapping with the DRM core code, but
901782b2
CW		 1346 * relinquish ownership of the pages back to the system.
1347 *
1348 * It is vital that we remove the page mapping if we have mapped a tiled
1349 * object through the GTT and then lose the fence register due to
1350 * resource pressure. Similarly if the object has been moved out of the
1351 * aperture, than pages mapped into userspace must be revoked. Removing the
1352 * mapping will then trigger a page fault on the next user access, allowing
1353 * fixup by i915_gem_fault().
1354 */
d05ca301 1355void
901782b2
CW		 1356i915_gem_release_mmap(struct drm_gem_object *obj)
1357{
1358 struct drm_device *dev = obj->dev;
23010e43 1359 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
901782b2
CW		 1360
1361 if (dev->dev_mapping)
1362 unmap_mapping_range(dev->dev_mapping,
1363 obj_priv->mmap_offset, obj->size, 1);
1364}
1365
ab00b3e5
JB		 1366static void
1367i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1368{
1369 struct drm_device *dev = obj->dev;
23010e43 1370 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ab00b3e5
JB		 1371 struct drm_gem_mm *mm = dev->mm_private;
1372 struct drm_map_list *list;
1373
1374 list = &obj->map_list;
1375 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1376
1377 if (list->file_offset_node) {
1378 drm_mm_put_block(list->file_offset_node);
1379 list->file_offset_node = NULL;
1380 }
1381
1382 if (list->map) {
9a298b2a 1383 kfree(list->map);
ab00b3e5
JB		 1384 list->map = NULL;
1385 }
1386
1387 obj_priv->mmap_offset = 0;
1388}
1389
de151cf6
JB		 1390/**
1391 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1392 * @obj: object to check
1393 *
1394 * Return the required GTT alignment for an object, taking into account
1395 * potential fence register mapping if needed.
1396 */
1397static uint32_t
1398i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1399{
1400 struct drm_device *dev = obj->dev;
23010e43 1401 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 1402 int start, i;
1403
1404 /*
1405 * Minimum alignment is 4k (GTT page size), but might be greater
1406 * if a fence register is needed for the object.
1407 */
a6c45cf0 1408 if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
de151cf6
JB		 1409 return 4096;
1410
1411 /*
1412 * Previous chips need to be aligned to the size of the smallest
1413 * fence register that can contain the object.
1414 */
a6c45cf0 1415 if (INTEL_INFO(dev)->gen == 3)
de151cf6
JB		 1416 start = 1024*1024;
1417 else
1418 start = 512*1024;
1419
1420 for (i = start; i < obj->size; i <<= 1)
1421 ;
1422
1423 return i;
1424}
1425
1426/**
1427 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1428 * @dev: DRM device
1429 * @data: GTT mapping ioctl data
1430 * @file_priv: GEM object info
1431 *
1432 * Simply returns the fake offset to userspace so it can mmap it.
1433 * The mmap call will end up in drm_gem_mmap(), which will set things
1434 * up so we can get faults in the handler above.
1435 *
1436 * The fault handler will take care of binding the object into the GTT
1437 * (since it may have been evicted to make room for something), allocating
1438 * a fence register, and mapping the appropriate aperture address into
1439 * userspace.
1440 */
1441int
1442i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1443 struct drm_file *file_priv)
1444{
1445 struct drm_i915_gem_mmap_gtt *args = data;
de151cf6
JB		 1446 struct drm_gem_object *obj;
1447 struct drm_i915_gem_object *obj_priv;
1448 int ret;
1449
1450 if (!(dev->driver->driver_features & DRIVER_GEM))
1451 return -ENODEV;
1452
1453 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1454 if (obj == NULL)
bf79cb91 1455 return -ENOENT;
de151cf6 1456
76c1dec1
CW		 1457 ret = i915_mutex_lock_interruptible(dev);
1458 if (ret) {
1459 drm_gem_object_unreference_unlocked(obj);
1460 return ret;
1461 }
de151cf6 1462
23010e43 1463 obj_priv = to_intel_bo(obj);
de151cf6 1464
ab18282d
CW		 1465 if (obj_priv->madv != I915_MADV_WILLNEED) {
1466 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1467 drm_gem_object_unreference(obj);
1468 mutex_unlock(&dev->struct_mutex);
1469 return -EINVAL;
1470 }
1471
1472
de151cf6
JB		 1473 if (!obj_priv->mmap_offset) {
1474 ret = i915_gem_create_mmap_offset(obj);
13af1062
CW		 1475 if (ret) {
1476 drm_gem_object_unreference(obj);
1477 mutex_unlock(&dev->struct_mutex);
de151cf6 1478 return ret;
13af1062 1479 }
de151cf6
JB		 1480 }
1481
1482 args->offset = obj_priv->mmap_offset;
1483
de151cf6
JB		 1484 /*
1485 * Pull it into the GTT so that we have a page list (makes the
1486 * initial fault faster and any subsequent flushing possible).
1487 */
1488 if (!obj_priv->agp_mem) {
e67b8ce1 1489 ret = i915_gem_object_bind_to_gtt(obj, 0);
de151cf6
JB		 1490 if (ret) {
1491 drm_gem_object_unreference(obj);
1492 mutex_unlock(&dev->struct_mutex);
1493 return ret;
1494 }
de151cf6
JB		 1495 }
1496
1497 drm_gem_object_unreference(obj);
1498 mutex_unlock(&dev->struct_mutex);
1499
1500 return 0;
1501}
1502
6911a9b8 1503void
856fa198 1504i915_gem_object_put_pages(struct drm_gem_object *obj)
673a394b 1505{
23010e43 1506 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 1507 int page_count = obj->size / PAGE_SIZE;
1508 int i;
1509
856fa198 1510 BUG_ON(obj_priv->pages_refcount == 0);
bb6baf76 1511 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
673a394b 1512
856fa198
EA		 1513 if (--obj_priv->pages_refcount != 0)
1514 return;
673a394b 1515
280b713b
EA		 1516 if (obj_priv->tiling_mode != I915_TILING_NONE)
1517 i915_gem_object_save_bit_17_swizzle(obj);
1518
3ef94daa 1519 if (obj_priv->madv == I915_MADV_DONTNEED)
13a05fd9 1520 obj_priv->dirty = 0;
3ef94daa
CW		 1521
1522 for (i = 0; i < page_count; i++) {
3ef94daa
CW		 1523 if (obj_priv->dirty)
1524 set_page_dirty(obj_priv->pages[i]);
1525
1526 if (obj_priv->madv == I915_MADV_WILLNEED)
856fa198 1527 mark_page_accessed(obj_priv->pages[i]);
3ef94daa
CW		 1528
1529 page_cache_release(obj_priv->pages[i]);
1530 }
673a394b
EA		 1531 obj_priv->dirty = 0;
1532
8e7d2b2c 1533 drm_free_large(obj_priv->pages);
856fa198 1534 obj_priv->pages = NULL;
673a394b
EA		 1535}
1536
a56ba56c
CW		 1537static uint32_t
1538i915_gem_next_request_seqno(struct drm_device *dev,
1539 struct intel_ring_buffer *ring)
1540{
1541 drm_i915_private_t *dev_priv = dev->dev_private;
1542
1543 ring->outstanding_lazy_request = true;
1544 return dev_priv->next_seqno;
1545}
1546
673a394b 1547static void
617dbe27 1548i915_gem_object_move_to_active(struct drm_gem_object *obj,
852835f3 1549 struct intel_ring_buffer *ring)
673a394b 1550{
a56ba56c 1551 struct drm_device *dev = obj->dev;
23010e43 1552 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
a56ba56c 1553 uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
617dbe27 1554
852835f3
ZN		 1555 BUG_ON(ring == NULL);
1556 obj_priv->ring = ring;
673a394b
EA		 1557
1558 /* Add a reference if we're newly entering the active list. */
1559 if (!obj_priv->active) {
1560 drm_gem_object_reference(obj);
1561 obj_priv->active = 1;
1562 }
e35a41de 1563
673a394b 1564 /* Move from whatever list we were on to the tail of execution. */
852835f3 1565 list_move_tail(&obj_priv->list, &ring->active_list);
a56ba56c 1566 obj_priv->last_rendering_seqno = seqno;
673a394b
EA		 1567}
1568
ce44b0ea
EA		 1569static void
1570i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1571{
1572 struct drm_device *dev = obj->dev;
1573 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1574 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ce44b0ea
EA		 1575
1576 BUG_ON(!obj_priv->active);
1577 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1578 obj_priv->last_rendering_seqno = 0;
1579}
673a394b 1580
963b4836
CW		 1581/* Immediately discard the backing storage */
1582static void
1583i915_gem_object_truncate(struct drm_gem_object *obj)
1584{
23010e43 1585 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
bb6baf76 1586 struct inode *inode;
963b4836 1587
ae9fed6b
CW		 1588 /* Our goal here is to return as much of the memory as
1589 * is possible back to the system as we are called from OOM.
1590 * To do this we must instruct the shmfs to drop all of its
1591 * backing pages, *now*. Here we mirror the actions taken
1592 * when by shmem_delete_inode() to release the backing store.
1593 */
bb6baf76 1594 inode = obj->filp->f_path.dentry->d_inode;
ae9fed6b
CW		 1595 truncate_inode_pages(inode->i_mapping, 0);
1596 if (inode->i_op->truncate_range)
1597 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
bb6baf76
CW		 1598
1599 obj_priv->madv = __I915_MADV_PURGED;
963b4836
CW		 1600}
1601
1602static inline int
1603i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1604{
1605 return obj_priv->madv == I915_MADV_DONTNEED;
1606}
1607
673a394b
EA		 1608static void
1609i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1610{
1611 struct drm_device *dev = obj->dev;
1612 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1613 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 1614
1615 i915_verify_inactive(dev, __FILE__, __LINE__);
1616 if (obj_priv->pin_count != 0)
f13d3f73 1617 list_move_tail(&obj_priv->list, &dev_priv->mm.pinned_list);
673a394b
EA		 1618 else
1619 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1620
99fcb766
DV		 1621 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1622
ce44b0ea 1623 obj_priv->last_rendering_seqno = 0;
852835f3 1624 obj_priv->ring = NULL;
673a394b
EA		 1625 if (obj_priv->active) {
1626 obj_priv->active = 0;
1627 drm_gem_object_unreference(obj);
1628 }
1629 i915_verify_inactive(dev, __FILE__, __LINE__);
1630}
1631
9220434a 1632static void
63560396 1633i915_gem_process_flushing_list(struct drm_device *dev,
8a1a49f9 1634 uint32_t flush_domains,
852835f3 1635 struct intel_ring_buffer *ring)
63560396
DV		 1636{
1637 drm_i915_private_t *dev_priv = dev->dev_private;
1638 struct drm_i915_gem_object *obj_priv, *next;
1639
1640 list_for_each_entry_safe(obj_priv, next,
1641 &dev_priv->mm.gpu_write_list,
1642 gpu_write_list) {
a8089e84 1643 struct drm_gem_object *obj = &obj_priv->base;
63560396 1644
2b6efaa4
CW		 1645 if (obj->write_domain & flush_domains &&
1646 obj_priv->ring == ring) {
63560396
DV		 1647 uint32_t old_write_domain = obj->write_domain;
1648
1649 obj->write_domain = 0;
1650 list_del_init(&obj_priv->gpu_write_list);
617dbe27 1651 i915_gem_object_move_to_active(obj, ring);
63560396
DV		 1652
1653 /* update the fence lru list */
007cc8ac
DV		 1654 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1655 struct drm_i915_fence_reg *reg =
1656 &dev_priv->fence_regs[obj_priv->fence_reg];
1657 list_move_tail(&reg->lru_list,
63560396 1658 &dev_priv->mm.fence_list);
007cc8ac 1659 }
63560396
DV		 1660
1661 trace_i915_gem_object_change_domain(obj,
1662 obj->read_domains,
1663 old_write_domain);
1664 }
1665 }
1666}
8187a2b7 1667
5a5a0c64 1668uint32_t
8a1a49f9 1669i915_add_request(struct drm_device *dev,
f787a5f5 1670 struct drm_file *file,
8dc5d147 1671 struct drm_i915_gem_request *request,
8a1a49f9 1672 struct intel_ring_buffer *ring)
673a394b
EA		 1673{
1674 drm_i915_private_t *dev_priv = dev->dev_private;
f787a5f5 1675 struct drm_i915_file_private *file_priv = NULL;
673a394b
EA		 1676 uint32_t seqno;
1677 int was_empty;
673a394b 1678
f787a5f5
CW		 1679 if (file != NULL)
1680 file_priv = file->driver_priv;
b962442e 1681
8dc5d147
CW		 1682 if (request == NULL) {
1683 request = kzalloc(sizeof(*request), GFP_KERNEL);
1684 if (request == NULL)
1685 return 0;
1686 }
673a394b 1687
f787a5f5 1688 seqno = ring->add_request(dev, ring, 0);
a56ba56c 1689 ring->outstanding_lazy_request = false;
673a394b
EA		 1690
1691 request->seqno = seqno;
852835f3 1692 request->ring = ring;
673a394b 1693 request->emitted_jiffies = jiffies;
852835f3
ZN		 1694 was_empty = list_empty(&ring->request_list);
1695 list_add_tail(&request->list, &ring->request_list);
1696
f787a5f5 1697 if (file_priv) {
1c25595f 1698 spin_lock(&file_priv->mm.lock);
f787a5f5 1699 request->file_priv = file_priv;
b962442e 1700 list_add_tail(&request->client_list,
f787a5f5 1701 &file_priv->mm.request_list);
1c25595f 1702 spin_unlock(&file_priv->mm.lock);
b962442e 1703 }
673a394b 1704
f65d9421 1705 if (!dev_priv->mm.suspended) {
b3b079db
CW		 1706 mod_timer(&dev_priv->hangcheck_timer,
1707 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
f65d9421 1708 if (was_empty)
b3b079db
CW		 1709 queue_delayed_work(dev_priv->wq,
1710 &dev_priv->mm.retire_work, HZ);
f65d9421 1711 }
673a394b
EA		 1712 return seqno;
1713}
1714
1715/**
1716 * Command execution barrier
1717 *
1718 * Ensures that all commands in the ring are finished
1719 * before signalling the CPU
1720 */
8a1a49f9 1721static void
852835f3 1722i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
673a394b 1723{
673a394b 1724 uint32_t flush_domains = 0;
673a394b
EA		 1725
1726 /* The sampler always gets flushed on i965 (sigh) */
a6c45cf0 1727 if (INTEL_INFO(dev)->gen >= 4)
673a394b 1728 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
852835f3
ZN		 1729
1730 ring->flush(dev, ring,
1731 I915_GEM_DOMAIN_COMMAND, flush_domains);
673a394b
EA		 1732}
1733
f787a5f5
CW		 1734static inline void
1735i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
673a394b 1736{
1c25595f
CW		 1737 struct drm_i915_file_private *file_priv = request->file_priv;
1738
1739 if (!file_priv)
1740 return;
1741
1742 spin_lock(&file_priv->mm.lock);
1743 list_del(&request->client_list);
1744 request->file_priv = NULL;
1745 spin_unlock(&file_priv->mm.lock);
673a394b
EA		 1746}
1747
dfaae392
CW		 1748static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1749 struct intel_ring_buffer *ring)
9375e446 1750{
dfaae392
CW		 1751 while (!list_empty(&ring->request_list)) {
1752 struct drm_i915_gem_request *request;
9375e446 1753
dfaae392
CW		 1754 request = list_first_entry(&ring->request_list,
1755 struct drm_i915_gem_request,
1756 list);
1757
1758 list_del(&request->list);
f787a5f5 1759 i915_gem_request_remove_from_client(request);
dfaae392
CW		 1760 kfree(request);
1761 }
1762
1763 while (!list_empty(&ring->active_list)) {
9375e446
CW		 1764 struct drm_i915_gem_object *obj_priv;
1765
dfaae392 1766 obj_priv = list_first_entry(&ring->active_list,
9375e446
CW		 1767 struct drm_i915_gem_object,
1768 list);
1769
1770 obj_priv->base.write_domain = 0;
dfaae392 1771 list_del_init(&obj_priv->gpu_write_list);
9375e446
CW		 1772 i915_gem_object_move_to_inactive(&obj_priv->base);
1773 }
1774}
1775
dfaae392 1776void i915_gem_reset_lists(struct drm_device *dev)
77f01230
CW		 1777{
1778 struct drm_i915_private *dev_priv = dev->dev_private;
1779 struct drm_i915_gem_object *obj_priv;
1780
dfaae392
CW		 1781 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1782 if (HAS_BSD(dev))
1783 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1784
1785 /* Remove anything from the flushing lists. The GPU cache is likely
1786 * to be lost on reset along with the data, so simply move the
1787 * lost bo to the inactive list.
1788 */
1789 while (!list_empty(&dev_priv->mm.flushing_list)) {
1790 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1791 struct drm_i915_gem_object,
1792 list);
1793
1794 obj_priv->base.write_domain = 0;
1795 list_del_init(&obj_priv->gpu_write_list);
1796 i915_gem_object_move_to_inactive(&obj_priv->base);
1797 }
1798
1799 /* Move everything out of the GPU domains to ensure we do any
1800 * necessary invalidation upon reuse.
1801 */
77f01230
CW		 1802 list_for_each_entry(obj_priv,
1803 &dev_priv->mm.inactive_list,
1804 list)
1805 {
1806 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1807 }
1808}
1809
673a394b
EA		 1810/**
1811 * This function clears the request list as sequence numbers are passed.
1812 */
b09a1fec
CW		 1813static void
1814i915_gem_retire_requests_ring(struct drm_device *dev,
1815 struct intel_ring_buffer *ring)
673a394b
EA		 1816{
1817 drm_i915_private_t *dev_priv = dev->dev_private;
1818 uint32_t seqno;
1819
b84d5f0c
CW		 1820 if (!ring->status_page.page_addr ||
1821 list_empty(&ring->request_list))
6c0594a3
KW		 1822 return;
1823
f787a5f5 1824 seqno = ring->get_seqno(dev, ring);
852835f3 1825 while (!list_empty(&ring->request_list)) {
673a394b 1826 struct drm_i915_gem_request *request;
673a394b 1827
852835f3 1828 request = list_first_entry(&ring->request_list,
673a394b
EA		 1829 struct drm_i915_gem_request,
1830 list);
673a394b 1831
dfaae392 1832 if (!i915_seqno_passed(seqno, request->seqno))
b84d5f0c
CW		 1833 break;
1834
1835 trace_i915_gem_request_retire(dev, request->seqno);
1836
1837 list_del(&request->list);
f787a5f5 1838 i915_gem_request_remove_from_client(request);
b84d5f0c
CW		 1839 kfree(request);
1840 }
1841
1842 /* Move any buffers on the active list that are no longer referenced
1843 * by the ringbuffer to the flushing/inactive lists as appropriate.
1844 */
1845 while (!list_empty(&ring->active_list)) {
1846 struct drm_gem_object *obj;
1847 struct drm_i915_gem_object *obj_priv;
1848
1849 obj_priv = list_first_entry(&ring->active_list,
1850 struct drm_i915_gem_object,
1851 list);
673a394b 1852
dfaae392 1853 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
673a394b 1854 break;
b84d5f0c
CW		 1855
1856 obj = &obj_priv->base;
b84d5f0c
CW		 1857 if (obj->write_domain != 0)
1858 i915_gem_object_move_to_flushing(obj);
1859 else
1860 i915_gem_object_move_to_inactive(obj);
673a394b 1861 }
9d34e5db
CW		 1862
1863 if (unlikely (dev_priv->trace_irq_seqno &&
1864 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
8187a2b7 1865 ring->user_irq_put(dev, ring);
9d34e5db
CW		 1866 dev_priv->trace_irq_seqno = 0;
1867 }
673a394b
EA		 1868}
1869
b09a1fec
CW		 1870void
1871i915_gem_retire_requests(struct drm_device *dev)
1872{
1873 drm_i915_private_t *dev_priv = dev->dev_private;
1874
be72615b
CW		 1875 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1876 struct drm_i915_gem_object *obj_priv, *tmp;
1877
1878 /* We must be careful that during unbind() we do not
1879 * accidentally infinitely recurse into retire requests.
1880 * Currently:
1881 * retire -> free -> unbind -> wait -> retire_ring
1882 */
1883 list_for_each_entry_safe(obj_priv, tmp,
1884 &dev_priv->mm.deferred_free_list,
1885 list)
1886 i915_gem_free_object_tail(&obj_priv->base);
1887 }
1888
b09a1fec
CW		 1889 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1890 if (HAS_BSD(dev))
1891 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1892}
1893
75ef9da2 1894static void
673a394b
EA		 1895i915_gem_retire_work_handler(struct work_struct *work)
1896{
1897 drm_i915_private_t *dev_priv;
1898 struct drm_device *dev;
1899
1900 dev_priv = container_of(work, drm_i915_private_t,
1901 mm.retire_work.work);
1902 dev = dev_priv->dev;
1903
1904 mutex_lock(&dev->struct_mutex);
b09a1fec 1905 i915_gem_retire_requests(dev);
d1b851fc 1906
6dbe2772 1907 if (!dev_priv->mm.suspended &&
d1b851fc
ZN		 1908 (!list_empty(&dev_priv->render_ring.request_list) ||
1909 (HAS_BSD(dev) &&
1910 !list_empty(&dev_priv->bsd_ring.request_list))))
9c9fe1f8 1911 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
673a394b
EA		 1912 mutex_unlock(&dev->struct_mutex);
1913}
1914
5a5a0c64 1915int
852835f3 1916i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
8a1a49f9 1917 bool interruptible, struct intel_ring_buffer *ring)
673a394b
EA		 1918{
1919 drm_i915_private_t *dev_priv = dev->dev_private;
802c7eb6 1920 u32 ier;
673a394b
EA		 1921 int ret = 0;
1922
1923 BUG_ON(seqno == 0);
1924
30dbf0c0
CW		 1925 if (atomic_read(&dev_priv->mm.wedged))
1926 return -EAGAIN;
1927
a56ba56c 1928 if (ring->outstanding_lazy_request) {
8dc5d147 1929 seqno = i915_add_request(dev, NULL, NULL, ring);
e35a41de
DV		 1930 if (seqno == 0)
1931 return -ENOMEM;
1932 }
a56ba56c 1933 BUG_ON(seqno == dev_priv->next_seqno);
e35a41de 1934
f787a5f5 1935 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
bad720ff 1936 if (HAS_PCH_SPLIT(dev))
036a4a7d
ZW		 1937 ier = I915_READ(DEIER) | I915_READ(GTIER);
1938 else
1939 ier = I915_READ(IER);
802c7eb6
JB		 1940 if (!ier) {
1941 DRM_ERROR("something (likely vbetool) disabled "
1942 "interrupts, re-enabling\n");
1943 i915_driver_irq_preinstall(dev);
1944 i915_driver_irq_postinstall(dev);
1945 }
1946
1c5d22f7
CW		 1947 trace_i915_gem_request_wait_begin(dev, seqno);
1948
852835f3 1949 ring->waiting_gem_seqno = seqno;
8187a2b7 1950 ring->user_irq_get(dev, ring);
48764bf4 1951 if (interruptible)
852835f3
ZN		 1952 ret = wait_event_interruptible(ring->irq_queue,
1953 i915_seqno_passed(
f787a5f5 1954 ring->get_seqno(dev, ring), seqno)
852835f3 1955 || atomic_read(&dev_priv->mm.wedged));
48764bf4 1956 else
852835f3
ZN		 1957 wait_event(ring->irq_queue,
1958 i915_seqno_passed(
f787a5f5 1959 ring->get_seqno(dev, ring), seqno)
852835f3 1960 || atomic_read(&dev_priv->mm.wedged));
48764bf4 1961
8187a2b7 1962 ring->user_irq_put(dev, ring);
852835f3 1963 ring->waiting_gem_seqno = 0;
1c5d22f7
CW		 1964
1965 trace_i915_gem_request_wait_end(dev, seqno);
673a394b 1966 }
ba1234d1 1967 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0 1968 ret = -EAGAIN;
673a394b
EA		 1969
1970 if (ret && ret != -ERESTARTSYS)
8bff917c 1971 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
f787a5f5 1972 __func__, ret, seqno, ring->get_seqno(dev, ring),
8bff917c 1973 dev_priv->next_seqno);
673a394b
EA		 1974
1975 /* Directly dispatch request retiring. While we have the work queue
1976 * to handle this, the waiter on a request often wants an associated
1977 * buffer to have made it to the inactive list, and we would need
1978 * a separate wait queue to handle that.
1979 */
1980 if (ret == 0)
b09a1fec 1981 i915_gem_retire_requests_ring(dev, ring);
673a394b
EA		 1982
1983 return ret;
1984}
1985
48764bf4
DV		 1986/**
1987 * Waits for a sequence number to be signaled, and cleans up the
1988 * request and object lists appropriately for that event.
1989 */
1990static int
852835f3 1991i915_wait_request(struct drm_device *dev, uint32_t seqno,
a56ba56c 1992 struct intel_ring_buffer *ring)
48764bf4 1993{
852835f3 1994 return i915_do_wait_request(dev, seqno, 1, ring);
48764bf4
DV		 1995}
1996
20f0cd55 1997static void
9220434a 1998i915_gem_flush_ring(struct drm_device *dev,
c78ec30b 1999 struct drm_file *file_priv,
9220434a
CW		 2000 struct intel_ring_buffer *ring,
2001 uint32_t invalidate_domains,
2002 uint32_t flush_domains)
2003{
2004 ring->flush(dev, ring, invalidate_domains, flush_domains);
2005 i915_gem_process_flushing_list(dev, flush_domains, ring);
2006}
2007
8187a2b7
ZN		 2008static void
2009i915_gem_flush(struct drm_device *dev,
c78ec30b 2010 struct drm_file *file_priv,
8187a2b7 2011 uint32_t invalidate_domains,
9220434a
CW		 2012 uint32_t flush_domains,
2013 uint32_t flush_rings)
8187a2b7
ZN		 2014{
2015 drm_i915_private_t *dev_priv = dev->dev_private;
8bff917c 2016
8187a2b7
ZN		 2017 if (flush_domains & I915_GEM_DOMAIN_CPU)
2018 drm_agp_chipset_flush(dev);
8bff917c 2019
9220434a
CW		 2020 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2021 if (flush_rings & RING_RENDER)
c78ec30b 2022 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 2023 &dev_priv->render_ring,
2024 invalidate_domains, flush_domains);
2025 if (flush_rings & RING_BSD)
c78ec30b 2026 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 2027 &dev_priv->bsd_ring,
2028 invalidate_domains, flush_domains);
2029 }
8187a2b7
ZN		 2030}
2031
673a394b
EA		 2032/**
2033 * Ensures that all rendering to the object has completed and the object is
2034 * safe to unbind from the GTT or access from the CPU.
2035 */
2036static int
2cf34d7b
CW		 2037i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2038 bool interruptible)
673a394b
EA		 2039{
2040 struct drm_device *dev = obj->dev;
23010e43 2041 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2042 int ret;
2043
e47c68e9
EA		 2044 /* This function only exists to support waiting for existing rendering,
2045 * not for emitting required flushes.
673a394b 2046 */
e47c68e9 2047 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 2048
2049 /* If there is rendering queued on the buffer being evicted, wait for
2050 * it.
2051 */
2052 if (obj_priv->active) {
2cf34d7b
CW		 2053 ret = i915_do_wait_request(dev,
2054 obj_priv->last_rendering_seqno,
2055 interruptible,
2056 obj_priv->ring);
2057 if (ret)
673a394b
EA		 2058 return ret;
2059 }
2060
2061 return 0;
2062}
2063
2064/**
2065 * Unbinds an object from the GTT aperture.
2066 */
0f973f27 2067int
673a394b
EA		 2068i915_gem_object_unbind(struct drm_gem_object *obj)
2069{
2070 struct drm_device *dev = obj->dev;
23010e43 2071 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2072 int ret = 0;
2073
673a394b
EA		 2074 if (obj_priv->gtt_space == NULL)
2075 return 0;
2076
2077 if (obj_priv->pin_count != 0) {
2078 DRM_ERROR("Attempting to unbind pinned buffer\n");
2079 return -EINVAL;
2080 }
2081
5323fd04
EA		 2082 /* blow away mappings if mapped through GTT */
2083 i915_gem_release_mmap(obj);
2084
673a394b
EA		 2085 /* Move the object to the CPU domain to ensure that
2086 * any possible CPU writes while it's not in the GTT
2087 * are flushed when we go to remap it. This will
2088 * also ensure that all pending GPU writes are finished
2089 * before we unbind.
2090 */
e47c68e9 2091 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
8dc1775d 2092 if (ret == -ERESTARTSYS)
673a394b 2093 return ret;
8dc1775d
CW		 2094 /* Continue on if we fail due to EIO, the GPU is hung so we
2095 * should be safe and we need to cleanup or else we might
2096 * cause memory corruption through use-after-free.
2097 */
673a394b 2098
96b47b65
DV		 2099 /* release the fence reg _after_ flushing */
2100 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2101 i915_gem_clear_fence_reg(obj);
2102
673a394b
EA		 2103 if (obj_priv->agp_mem != NULL) {
2104 drm_unbind_agp(obj_priv->agp_mem);
2105 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2106 obj_priv->agp_mem = NULL;
2107 }
2108
856fa198 2109 i915_gem_object_put_pages(obj);
a32808c0 2110 BUG_ON(obj_priv->pages_refcount);
673a394b
EA		 2111
2112 if (obj_priv->gtt_space) {
2113 atomic_dec(&dev->gtt_count);
2114 atomic_sub(obj->size, &dev->gtt_memory);
2115
2116 drm_mm_put_block(obj_priv->gtt_space);
2117 obj_priv->gtt_space = NULL;
2118 }
2119
f13d3f73 2120 list_del_init(&obj_priv->list);
673a394b 2121
963b4836
CW		 2122 if (i915_gem_object_is_purgeable(obj_priv))
2123 i915_gem_object_truncate(obj);
2124
1c5d22f7
CW		 2125 trace_i915_gem_object_unbind(obj);
2126
8dc1775d 2127 return ret;
673a394b
EA		 2128}
2129
a56ba56c
CW		 2130static int i915_ring_idle(struct drm_device *dev,
2131 struct intel_ring_buffer *ring)
2132{
2133 i915_gem_flush_ring(dev, NULL, ring,
2134 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2135 return i915_wait_request(dev,
2136 i915_gem_next_request_seqno(dev, ring),
2137 ring);
2138}
2139
b47eb4a2 2140int
4df2faf4
DV		 2141i915_gpu_idle(struct drm_device *dev)
2142{
2143 drm_i915_private_t *dev_priv = dev->dev_private;
2144 bool lists_empty;
852835f3 2145 int ret;
4df2faf4 2146
d1b851fc
ZN		 2147 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2148 list_empty(&dev_priv->render_ring.active_list) &&
2149 (!HAS_BSD(dev) ||
2150 list_empty(&dev_priv->bsd_ring.active_list)));
4df2faf4
DV		 2151 if (lists_empty)
2152 return 0;
2153
2154 /* Flush everything onto the inactive list. */
a56ba56c 2155 ret = i915_ring_idle(dev, &dev_priv->render_ring);
8a1a49f9
DV		 2156 if (ret)
2157 return ret;
d1b851fc
ZN		 2158
2159 if (HAS_BSD(dev)) {
a56ba56c 2160 ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
d1b851fc
ZN		 2161 if (ret)
2162 return ret;
2163 }
2164
8a1a49f9 2165 return 0;
4df2faf4
DV		 2166}
2167
6911a9b8 2168int
4bdadb97
CW		 2169i915_gem_object_get_pages(struct drm_gem_object *obj,
2170 gfp_t gfpmask)
673a394b 2171{
23010e43 2172 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2173 int page_count, i;
2174 struct address_space *mapping;
2175 struct inode *inode;
2176 struct page *page;
673a394b 2177
778c3544
DV		 2178 BUG_ON(obj_priv->pages_refcount
2179 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2180
856fa198 2181 if (obj_priv->pages_refcount++ != 0)
673a394b
EA		 2182 return 0;
2183
2184 /* Get the list of pages out of our struct file. They'll be pinned
2185 * at this point until we release them.
2186 */
2187 page_count = obj->size / PAGE_SIZE;
856fa198 2188 BUG_ON(obj_priv->pages != NULL);
8e7d2b2c 2189 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
856fa198 2190 if (obj_priv->pages == NULL) {
856fa198 2191 obj_priv->pages_refcount--;
673a394b
EA		 2192 return -ENOMEM;
2193 }
2194
2195 inode = obj->filp->f_path.dentry->d_inode;
2196 mapping = inode->i_mapping;
2197 for (i = 0; i < page_count; i++) {
4bdadb97 2198 page = read_cache_page_gfp(mapping, i,
985b823b 2199 GFP_HIGHUSER |
4bdadb97 2200 __GFP_COLD |
cd9f040d 2201 __GFP_RECLAIMABLE |
4bdadb97 2202 gfpmask);
1f2b1013
CW		 2203 if (IS_ERR(page))
2204 goto err_pages;
2205
856fa198 2206 obj_priv->pages[i] = page;
673a394b 2207 }
280b713b
EA		 2208
2209 if (obj_priv->tiling_mode != I915_TILING_NONE)
2210 i915_gem_object_do_bit_17_swizzle(obj);
2211
673a394b 2212 return 0;
1f2b1013
CW		 2213
2214err_pages:
2215 while (i--)
2216 page_cache_release(obj_priv->pages[i]);
2217
2218 drm_free_large(obj_priv->pages);
2219 obj_priv->pages = NULL;
2220 obj_priv->pages_refcount--;
2221 return PTR_ERR(page);
673a394b
EA		 2222}
2223
4e901fdc
EA		 2224static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2225{
2226 struct drm_gem_object *obj = reg->obj;
2227 struct drm_device *dev = obj->dev;
2228 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2229 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4e901fdc
EA		 2230 int regnum = obj_priv->fence_reg;
2231 uint64_t val;
2232
2233 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2234 0xfffff000) << 32;
2235 val |= obj_priv->gtt_offset & 0xfffff000;
2236 val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2237 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2238
2239 if (obj_priv->tiling_mode == I915_TILING_Y)
2240 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2241 val |= I965_FENCE_REG_VALID;
2242
2243 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2244}
2245
de151cf6
JB		 2246static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2247{
2248 struct drm_gem_object *obj = reg->obj;
2249 struct drm_device *dev = obj->dev;
2250 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2251 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 2252 int regnum = obj_priv->fence_reg;
2253 uint64_t val;
2254
2255 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2256 0xfffff000) << 32;
2257 val |= obj_priv->gtt_offset & 0xfffff000;
2258 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2259 if (obj_priv->tiling_mode == I915_TILING_Y)
2260 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2261 val |= I965_FENCE_REG_VALID;
2262
2263 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2264}
2265
2266static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2267{
2268 struct drm_gem_object *obj = reg->obj;
2269 struct drm_device *dev = obj->dev;
2270 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2271 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2272 int regnum = obj_priv->fence_reg;
0f973f27 2273 int tile_width;
dc529a4f 2274 uint32_t fence_reg, val;
de151cf6
JB		 2275 uint32_t pitch_val;
2276
2277 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2278 (obj_priv->gtt_offset & (obj->size - 1))) {
f06da264 2279 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
0f973f27 2280 __func__, obj_priv->gtt_offset, obj->size);
de151cf6
JB		 2281 return;
2282 }
2283
0f973f27
JB		 2284 if (obj_priv->tiling_mode == I915_TILING_Y &&
2285 HAS_128_BYTE_Y_TILING(dev))
2286 tile_width = 128;
de151cf6 2287 else
0f973f27
JB		 2288 tile_width = 512;
2289
2290 /* Note: pitch better be a power of two tile widths */
2291 pitch_val = obj_priv->stride / tile_width;
2292 pitch_val = ffs(pitch_val) - 1;
de151cf6 2293
c36a2a6d
DV		 2294 if (obj_priv->tiling_mode == I915_TILING_Y &&
2295 HAS_128_BYTE_Y_TILING(dev))
2296 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2297 else
2298 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2299
de151cf6
JB		 2300 val = obj_priv->gtt_offset;
2301 if (obj_priv->tiling_mode == I915_TILING_Y)
2302 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2303 val |= I915_FENCE_SIZE_BITS(obj->size);
2304 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2305 val |= I830_FENCE_REG_VALID;
2306
dc529a4f
EA		 2307 if (regnum < 8)
2308 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2309 else
2310 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2311 I915_WRITE(fence_reg, val);
de151cf6
JB		 2312}
2313
2314static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2315{
2316 struct drm_gem_object *obj = reg->obj;
2317 struct drm_device *dev = obj->dev;
2318 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2319 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 2320 int regnum = obj_priv->fence_reg;
2321 uint32_t val;
2322 uint32_t pitch_val;
8d7773a3 2323 uint32_t fence_size_bits;
de151cf6 2324
8d7773a3 2325 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
de151cf6 2326 (obj_priv->gtt_offset & (obj->size - 1))) {
8d7773a3 2327 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
0f973f27 2328 __func__, obj_priv->gtt_offset);
de151cf6
JB		 2329 return;
2330 }
2331
e76a16de
EA		 2332 pitch_val = obj_priv->stride / 128;
2333 pitch_val = ffs(pitch_val) - 1;
2334 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2335
de151cf6
JB		 2336 val = obj_priv->gtt_offset;
2337 if (obj_priv->tiling_mode == I915_TILING_Y)
2338 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
8d7773a3
DV		 2339 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2340 WARN_ON(fence_size_bits & ~0x00000f00);
2341 val |= fence_size_bits;
de151cf6
JB		 2342 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2343 val |= I830_FENCE_REG_VALID;
2344
2345 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
de151cf6
JB		 2346}
2347
2cf34d7b
CW		 2348static int i915_find_fence_reg(struct drm_device *dev,
2349 bool interruptible)
ae3db24a
DV		 2350{
2351 struct drm_i915_fence_reg *reg = NULL;
2352 struct drm_i915_gem_object *obj_priv = NULL;
2353 struct drm_i915_private *dev_priv = dev->dev_private;
2354 struct drm_gem_object *obj = NULL;
2355 int i, avail, ret;
2356
2357 /* First try to find a free reg */
2358 avail = 0;
2359 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2360 reg = &dev_priv->fence_regs[i];
2361 if (!reg->obj)
2362 return i;
2363
23010e43 2364 obj_priv = to_intel_bo(reg->obj);
ae3db24a
DV		 2365 if (!obj_priv->pin_count)
2366 avail++;
2367 }
2368
2369 if (avail == 0)
2370 return -ENOSPC;
2371
2372 /* None available, try to steal one or wait for a user to finish */
2373 i = I915_FENCE_REG_NONE;
007cc8ac
DV		 2374 list_for_each_entry(reg, &dev_priv->mm.fence_list,
2375 lru_list) {
2376 obj = reg->obj;
2377 obj_priv = to_intel_bo(obj);
ae3db24a
DV		 2378
2379 if (obj_priv->pin_count)
2380 continue;
2381
2382 /* found one! */
2383 i = obj_priv->fence_reg;
2384 break;
2385 }
2386
2387 BUG_ON(i == I915_FENCE_REG_NONE);
2388
2389 /* We only have a reference on obj from the active list. put_fence_reg
2390 * might drop that one, causing a use-after-free in it. So hold a
2391 * private reference to obj like the other callers of put_fence_reg
2392 * (set_tiling ioctl) do. */
2393 drm_gem_object_reference(obj);
2cf34d7b 2394 ret = i915_gem_object_put_fence_reg(obj, interruptible);
ae3db24a
DV		 2395 drm_gem_object_unreference(obj);
2396 if (ret != 0)
2397 return ret;
2398
2399 return i;
2400}
2401
de151cf6
JB		 2402/**
2403 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2404 * @obj: object to map through a fence reg
2405 *
2406 * When mapping objects through the GTT, userspace wants to be able to write
2407 * to them without having to worry about swizzling if the object is tiled.
2408 *
2409 * This function walks the fence regs looking for a free one for @obj,
2410 * stealing one if it can't find any.
2411 *
2412 * It then sets up the reg based on the object's properties: address, pitch
2413 * and tiling format.
2414 */
8c4b8c3f 2415int
2cf34d7b
CW		 2416i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2417 bool interruptible)
de151cf6
JB		 2418{
2419 struct drm_device *dev = obj->dev;
79e53945 2420 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2421 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2422 struct drm_i915_fence_reg *reg = NULL;
ae3db24a 2423 int ret;
de151cf6 2424
a09ba7fa
EA		 2425 /* Just update our place in the LRU if our fence is getting used. */
2426 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
007cc8ac
DV		 2427 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2428 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
a09ba7fa
EA		 2429 return 0;
2430 }
2431
de151cf6
JB		 2432 switch (obj_priv->tiling_mode) {
2433 case I915_TILING_NONE:
2434 WARN(1, "allocating a fence for non-tiled object?\n");
2435 break;
2436 case I915_TILING_X:
0f973f27
JB		 2437 if (!obj_priv->stride)
2438 return -EINVAL;
2439 WARN((obj_priv->stride & (512 - 1)),
2440 "object 0x%08x is X tiled but has non-512B pitch\n",
2441 obj_priv->gtt_offset);
de151cf6
JB		 2442 break;
2443 case I915_TILING_Y:
0f973f27
JB		 2444 if (!obj_priv->stride)
2445 return -EINVAL;
2446 WARN((obj_priv->stride & (128 - 1)),
2447 "object 0x%08x is Y tiled but has non-128B pitch\n",
2448 obj_priv->gtt_offset);
de151cf6
JB		 2449 break;
2450 }
2451
2cf34d7b 2452 ret = i915_find_fence_reg(dev, interruptible);
ae3db24a
DV		 2453 if (ret < 0)
2454 return ret;
de151cf6 2455
ae3db24a
DV		 2456 obj_priv->fence_reg = ret;
2457 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
007cc8ac 2458 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
a09ba7fa 2459
de151cf6
JB		 2460 reg->obj = obj;
2461
e259befd
CW		 2462 switch (INTEL_INFO(dev)->gen) {
2463 case 6:
4e901fdc 2464 sandybridge_write_fence_reg(reg);
e259befd
CW		 2465 break;
2466 case 5:
2467 case 4:
de151cf6 2468 i965_write_fence_reg(reg);
e259befd
CW		 2469 break;
2470 case 3:
de151cf6 2471 i915_write_fence_reg(reg);
e259befd
CW		 2472 break;
2473 case 2:
de151cf6 2474 i830_write_fence_reg(reg);
e259befd
CW		 2475 break;
2476 }
d9ddcb96 2477
ae3db24a
DV		 2478 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2479 obj_priv->tiling_mode);
1c5d22f7 2480
d9ddcb96 2481 return 0;
de151cf6
JB		 2482}
2483
2484/**
2485 * i915_gem_clear_fence_reg - clear out fence register info
2486 * @obj: object to clear
2487 *
2488 * Zeroes out the fence register itself and clears out the associated
2489 * data structures in dev_priv and obj_priv.
2490 */
2491static void
2492i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2493{
2494 struct drm_device *dev = obj->dev;
79e53945 2495 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2496 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
007cc8ac
DV		 2497 struct drm_i915_fence_reg *reg =
2498 &dev_priv->fence_regs[obj_priv->fence_reg];
e259befd 2499 uint32_t fence_reg;
de151cf6 2500
e259befd
CW		 2501 switch (INTEL_INFO(dev)->gen) {
2502 case 6:
4e901fdc
EA		 2503 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2504 (obj_priv->fence_reg * 8), 0);
e259befd
CW		 2505 break;
2506 case 5:
2507 case 4:
de151cf6 2508 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
e259befd
CW		 2509 break;
2510 case 3:
9b74f734 2511 if (obj_priv->fence_reg >= 8)
e259befd 2512 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
dc529a4f 2513 else
e259befd
CW		 2514 case 2:
2515 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
dc529a4f
EA		 2516
2517 I915_WRITE(fence_reg, 0);
e259befd 2518 break;
dc529a4f 2519 }
de151cf6 2520
007cc8ac 2521 reg->obj = NULL;
de151cf6 2522 obj_priv->fence_reg = I915_FENCE_REG_NONE;
007cc8ac 2523 list_del_init(&reg->lru_list);
de151cf6
JB		 2524}
2525
52dc7d32
CW		 2526/**
2527 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2528 * to the buffer to finish, and then resets the fence register.
2529 * @obj: tiled object holding a fence register.
2cf34d7b 2530 * @bool: whether the wait upon the fence is interruptible
52dc7d32
CW		 2531 *
2532 * Zeroes out the fence register itself and clears out the associated
2533 * data structures in dev_priv and obj_priv.
2534 */
2535int
2cf34d7b
CW		 2536i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2537 bool interruptible)
52dc7d32
CW		 2538{
2539 struct drm_device *dev = obj->dev;
53640e1d 2540 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2541 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
53640e1d 2542 struct drm_i915_fence_reg *reg;
52dc7d32
CW		 2543
2544 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2545 return 0;
2546
10ae9bd2
DV		 2547 /* If we've changed tiling, GTT-mappings of the object
2548 * need to re-fault to ensure that the correct fence register
2549 * setup is in place.
2550 */
2551 i915_gem_release_mmap(obj);
2552
52dc7d32
CW		 2553 /* On the i915, GPU access to tiled buffers is via a fence,
2554 * therefore we must wait for any outstanding access to complete
2555 * before clearing the fence.
2556 */
53640e1d
CW		 2557 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2558 if (reg->gpu) {
52dc7d32
CW		 2559 int ret;
2560
2cf34d7b 2561 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
0bc23aad
CW		 2562 if (ret)
2563 return ret;
2564
2cf34d7b 2565 ret = i915_gem_object_wait_rendering(obj, interruptible);
0bc23aad 2566 if (ret)
52dc7d32 2567 return ret;
53640e1d
CW		 2568
2569 reg->gpu = false;
52dc7d32
CW		 2570 }
2571
4a726612 2572 i915_gem_object_flush_gtt_write_domain(obj);
0bc23aad 2573 i915_gem_clear_fence_reg(obj);
52dc7d32
CW		 2574
2575 return 0;
2576}
2577
673a394b
EA		 2578/**
2579 * Finds free space in the GTT aperture and binds the object there.
2580 */
2581static int
2582i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2583{
2584 struct drm_device *dev = obj->dev;
2585 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2586 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 2587 struct drm_mm_node *free_space;
4bdadb97 2588 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
07f73f69 2589 int ret;
673a394b 2590
bb6baf76 2591 if (obj_priv->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2592 DRM_ERROR("Attempting to bind a purgeable object\n");
2593 return -EINVAL;
2594 }
2595
673a394b 2596 if (alignment == 0)
0f973f27 2597 alignment = i915_gem_get_gtt_alignment(obj);
8d7773a3 2598 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
673a394b
EA		 2599 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2600 return -EINVAL;
2601 }
2602
654fc607
CW		 2603 /* If the object is bigger than the entire aperture, reject it early
2604 * before evicting everything in a vain attempt to find space.
2605 */
2606 if (obj->size > dev->gtt_total) {
2607 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2608 return -E2BIG;
2609 }
2610
673a394b
EA		 2611 search_free:
2612 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2613 obj->size, alignment, 0);
2614 if (free_space != NULL) {
2615 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2616 alignment);
db3307a9 2617 if (obj_priv->gtt_space != NULL)
673a394b 2618 obj_priv->gtt_offset = obj_priv->gtt_space->start;
673a394b
EA		 2619 }
2620 if (obj_priv->gtt_space == NULL) {
2621 /* If the gtt is empty and we're still having trouble
2622 * fitting our object in, we're out of memory.
2623 */
0108a3ed 2624 ret = i915_gem_evict_something(dev, obj->size, alignment);
9731129c 2625 if (ret)
673a394b 2626 return ret;
9731129c 2627
673a394b
EA		 2628 goto search_free;
2629 }
2630
4bdadb97 2631 ret = i915_gem_object_get_pages(obj, gfpmask);
673a394b
EA		 2632 if (ret) {
2633 drm_mm_put_block(obj_priv->gtt_space);
2634 obj_priv->gtt_space = NULL;
07f73f69
CW		 2635
2636 if (ret == -ENOMEM) {
2637 /* first try to clear up some space from the GTT */
0108a3ed
DV		 2638 ret = i915_gem_evict_something(dev, obj->size,
2639 alignment);
07f73f69 2640 if (ret) {
07f73f69 2641 /* now try to shrink everyone else */
4bdadb97
CW		 2642 if (gfpmask) {
2643 gfpmask = 0;
2644 goto search_free;
07f73f69
CW		 2645 }
2646
2647 return ret;
2648 }
2649
2650 goto search_free;
2651 }
2652
673a394b
EA		 2653 return ret;
2654 }
2655
673a394b
EA		 2656 /* Create an AGP memory structure pointing at our pages, and bind it
2657 * into the GTT.
2658 */
2659 obj_priv->agp_mem = drm_agp_bind_pages(dev,
856fa198 2660 obj_priv->pages,
07f73f69 2661 obj->size >> PAGE_SHIFT,
ba1eb1d8
KP		 2662 obj_priv->gtt_offset,
2663 obj_priv->agp_type);
673a394b 2664 if (obj_priv->agp_mem == NULL) {
856fa198 2665 i915_gem_object_put_pages(obj);
673a394b
EA		 2666 drm_mm_put_block(obj_priv->gtt_space);
2667 obj_priv->gtt_space = NULL;
07f73f69 2668
0108a3ed 2669 ret = i915_gem_evict_something(dev, obj->size, alignment);
9731129c 2670 if (ret)
07f73f69 2671 return ret;
07f73f69
CW		 2672
2673 goto search_free;
673a394b
EA		 2674 }
2675 atomic_inc(&dev->gtt_count);
2676 atomic_add(obj->size, &dev->gtt_memory);
2677
bf1a1092
CW		 2678 /* keep track of bounds object by adding it to the inactive list */
2679 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2680
673a394b
EA		 2681 /* Assert that the object is not currently in any GPU domain. As it
2682 * wasn't in the GTT, there shouldn't be any way it could have been in
2683 * a GPU cache
2684 */
21d509e3
CW		 2685 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2686 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2687
1c5d22f7
CW		 2688 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2689
673a394b
EA		 2690 return 0;
2691}
2692
2693void
2694i915_gem_clflush_object(struct drm_gem_object *obj)
2695{
23010e43 2696 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2697
2698 /* If we don't have a page list set up, then we're not pinned
2699 * to GPU, and we can ignore the cache flush because it'll happen
2700 * again at bind time.
2701 */
856fa198 2702 if (obj_priv->pages == NULL)
673a394b
EA		 2703 return;
2704
1c5d22f7 2705 trace_i915_gem_object_clflush(obj);
cfa16a0d 2706
856fa198 2707 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
673a394b
EA		 2708}
2709
e47c68e9 2710/** Flushes any GPU write domain for the object if it's dirty. */
2dafb1e0 2711static int
ba3d8d74
DV		 2712i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2713 bool pipelined)
e47c68e9
EA		 2714{
2715 struct drm_device *dev = obj->dev;
1c5d22f7 2716 uint32_t old_write_domain;
e47c68e9
EA		 2717
2718 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2dafb1e0 2719 return 0;
e47c68e9
EA		 2720
2721 /* Queue the GPU write cache flushing we need. */
1c5d22f7 2722 old_write_domain = obj->write_domain;
c78ec30b 2723 i915_gem_flush_ring(dev, NULL,
9220434a
CW		 2724 to_intel_bo(obj)->ring,
2725 0, obj->write_domain);
48b956c5 2726 BUG_ON(obj->write_domain);
1c5d22f7
CW		 2727
2728 trace_i915_gem_object_change_domain(obj,
2729 obj->read_domains,
2730 old_write_domain);
ba3d8d74
DV		 2731
2732 if (pipelined)
2733 return 0;
2734
2cf34d7b 2735 return i915_gem_object_wait_rendering(obj, true);
e47c68e9
EA		 2736}
2737
2738/** Flushes the GTT write domain for the object if it's dirty. */
2739static void
2740i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2741{
1c5d22f7
CW		 2742 uint32_t old_write_domain;
2743
e47c68e9
EA		 2744 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2745 return;
2746
2747 /* No actual flushing is required for the GTT write domain. Writes
2748 * to it immediately go to main memory as far as we know, so there's
2749 * no chipset flush. It also doesn't land in render cache.
2750 */
1c5d22f7 2751 old_write_domain = obj->write_domain;
e47c68e9 2752 obj->write_domain = 0;
1c5d22f7
CW		 2753
2754 trace_i915_gem_object_change_domain(obj,
2755 obj->read_domains,
2756 old_write_domain);
e47c68e9
EA		 2757}
2758
2759/** Flushes the CPU write domain for the object if it's dirty. */
2760static void
2761i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2762{
2763 struct drm_device *dev = obj->dev;
1c5d22f7 2764 uint32_t old_write_domain;
e47c68e9
EA		 2765
2766 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2767 return;
2768
2769 i915_gem_clflush_object(obj);
2770 drm_agp_chipset_flush(dev);
1c5d22f7 2771 old_write_domain = obj->write_domain;
e47c68e9 2772 obj->write_domain = 0;
1c5d22f7
CW		 2773
2774 trace_i915_gem_object_change_domain(obj,
2775 obj->read_domains,
2776 old_write_domain);
e47c68e9
EA		 2777}
2778
2ef7eeaa
EA		 2779/**
2780 * Moves a single object to the GTT read, and possibly write domain.
2781 *
2782 * This function returns when the move is complete, including waiting on
2783 * flushes to occur.
2784 */
79e53945 2785int
2ef7eeaa
EA		 2786i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2787{
23010e43 2788 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 2789 uint32_t old_write_domain, old_read_domains;
e47c68e9 2790 int ret;
2ef7eeaa 2791
02354392
EA		 2792 /* Not valid to be called on unbound objects. */
2793 if (obj_priv->gtt_space == NULL)
2794 return -EINVAL;
2795
ba3d8d74 2796 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9
EA		 2797 if (ret != 0)
2798 return ret;
2799
7213342d 2800 i915_gem_object_flush_cpu_write_domain(obj);
1c5d22f7 2801
ba3d8d74 2802 if (write) {
2cf34d7b 2803 ret = i915_gem_object_wait_rendering(obj, true);
ba3d8d74
DV		 2804 if (ret)
2805 return ret;
ba3d8d74 2806 }
2ef7eeaa 2807
7213342d
CW		 2808 old_write_domain = obj->write_domain;
2809 old_read_domains = obj->read_domains;
2ef7eeaa 2810
e47c68e9
EA		 2811 /* It should now be out of any other write domains, and we can update
2812 * the domain values for our changes.
2813 */
2814 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2815 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2816 if (write) {
7213342d 2817 obj->read_domains = I915_GEM_DOMAIN_GTT;
e47c68e9
EA		 2818 obj->write_domain = I915_GEM_DOMAIN_GTT;
2819 obj_priv->dirty = 1;
2ef7eeaa
EA		 2820 }
2821
1c5d22f7
CW		 2822 trace_i915_gem_object_change_domain(obj,
2823 old_read_domains,
2824 old_write_domain);
2825
e47c68e9
EA		 2826 return 0;
2827}
2828
b9241ea3
ZW		 2829/*
2830 * Prepare buffer for display plane. Use uninterruptible for possible flush
2831 * wait, as in modesetting process we're not supposed to be interrupted.
2832 */
2833int
48b956c5
CW		 2834i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2835 bool pipelined)
b9241ea3 2836{
23010e43 2837 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ba3d8d74 2838 uint32_t old_read_domains;
b9241ea3
ZW		 2839 int ret;
2840
2841 /* Not valid to be called on unbound objects. */
2842 if (obj_priv->gtt_space == NULL)
2843 return -EINVAL;
2844
ced270fa 2845 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
48b956c5 2846 if (ret)
e35a41de 2847 return ret;
b9241ea3 2848
ced270fa
CW		 2849 /* Currently, we are always called from an non-interruptible context. */
2850 if (!pipelined) {
2851 ret = i915_gem_object_wait_rendering(obj, false);
2852 if (ret)
2853 return ret;
2854 }
2855
b118c1e3
CW		 2856 i915_gem_object_flush_cpu_write_domain(obj);
2857
b9241ea3 2858 old_read_domains = obj->read_domains;
c78ec30b 2859 obj->read_domains |= I915_GEM_DOMAIN_GTT;
b9241ea3
ZW		 2860
2861 trace_i915_gem_object_change_domain(obj,
2862 old_read_domains,
ba3d8d74 2863 obj->write_domain);
b9241ea3
ZW		 2864
2865 return 0;
2866}
2867
e47c68e9
EA		 2868/**
2869 * Moves a single object to the CPU read, and possibly write domain.
2870 *
2871 * This function returns when the move is complete, including waiting on
2872 * flushes to occur.
2873 */
2874static int
2875i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2876{
1c5d22f7 2877 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 2878 int ret;
2879
ba3d8d74 2880 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9
EA		 2881 if (ret != 0)
2882 return ret;
2ef7eeaa 2883
e47c68e9 2884 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 2885
e47c68e9
EA		 2886 /* If we have a partially-valid cache of the object in the CPU,
2887 * finish invalidating it and free the per-page flags.
2ef7eeaa 2888 */
e47c68e9 2889 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 2890
7213342d 2891 if (write) {
2cf34d7b 2892 ret = i915_gem_object_wait_rendering(obj, true);
7213342d
CW		 2893 if (ret)
2894 return ret;
2895 }
2896
1c5d22f7
CW		 2897 old_write_domain = obj->write_domain;
2898 old_read_domains = obj->read_domains;
2899
e47c68e9
EA		 2900 /* Flush the CPU cache if it's still invalid. */
2901 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 2902 i915_gem_clflush_object(obj);
2ef7eeaa 2903
e47c68e9 2904 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 2905 }
2906
2907 /* It should now be out of any other write domains, and we can update
2908 * the domain values for our changes.
2909 */
e47c68e9
EA		 2910 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2911
2912 /* If we're writing through the CPU, then the GPU read domains will
2913 * need to be invalidated at next use.
2914 */
2915 if (write) {
c78ec30b 2916 obj->read_domains = I915_GEM_DOMAIN_CPU;
e47c68e9
EA		 2917 obj->write_domain = I915_GEM_DOMAIN_CPU;
2918 }
2ef7eeaa 2919
1c5d22f7
CW		 2920 trace_i915_gem_object_change_domain(obj,
2921 old_read_domains,
2922 old_write_domain);
2923
2ef7eeaa
EA		 2924 return 0;
2925}
2926
673a394b
EA		 2927/*
2928 * Set the next domain for the specified object. This
2929 * may not actually perform the necessary flushing/invaliding though,
2930 * as that may want to be batched with other set_domain operations
2931 *
2932 * This is (we hope) the only really tricky part of gem. The goal
2933 * is fairly simple -- track which caches hold bits of the object
2934 * and make sure they remain coherent. A few concrete examples may
2935 * help to explain how it works. For shorthand, we use the notation
2936 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2937 * a pair of read and write domain masks.
2938 *
2939 * Case 1: the batch buffer
2940 *
2941 * 1. Allocated
2942 * 2. Written by CPU
2943 * 3. Mapped to GTT
2944 * 4. Read by GPU
2945 * 5. Unmapped from GTT
2946 * 6. Freed
2947 *
2948 * Let's take these a step at a time
2949 *
2950 * 1. Allocated
2951 * Pages allocated from the kernel may still have
2952 * cache contents, so we set them to (CPU, CPU) always.
2953 * 2. Written by CPU (using pwrite)
2954 * The pwrite function calls set_domain (CPU, CPU) and
2955 * this function does nothing (as nothing changes)
2956 * 3. Mapped by GTT
2957 * This function asserts that the object is not
2958 * currently in any GPU-based read or write domains
2959 * 4. Read by GPU
2960 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2961 * As write_domain is zero, this function adds in the
2962 * current read domains (CPU+COMMAND, 0).
2963 * flush_domains is set to CPU.
2964 * invalidate_domains is set to COMMAND
2965 * clflush is run to get data out of the CPU caches
2966 * then i915_dev_set_domain calls i915_gem_flush to
2967 * emit an MI_FLUSH and drm_agp_chipset_flush
2968 * 5. Unmapped from GTT
2969 * i915_gem_object_unbind calls set_domain (CPU, CPU)
2970 * flush_domains and invalidate_domains end up both zero
2971 * so no flushing/invalidating happens
2972 * 6. Freed
2973 * yay, done
2974 *
2975 * Case 2: The shared render buffer
2976 *
2977 * 1. Allocated
2978 * 2. Mapped to GTT
2979 * 3. Read/written by GPU
2980 * 4. set_domain to (CPU,CPU)
2981 * 5. Read/written by CPU
2982 * 6. Read/written by GPU
2983 *
2984 * 1. Allocated
2985 * Same as last example, (CPU, CPU)
2986 * 2. Mapped to GTT
2987 * Nothing changes (assertions find that it is not in the GPU)
2988 * 3. Read/written by GPU
2989 * execbuffer calls set_domain (RENDER, RENDER)
2990 * flush_domains gets CPU
2991 * invalidate_domains gets GPU
2992 * clflush (obj)
2993 * MI_FLUSH and drm_agp_chipset_flush
2994 * 4. set_domain (CPU, CPU)
2995 * flush_domains gets GPU
2996 * invalidate_domains gets CPU
2997 * wait_rendering (obj) to make sure all drawing is complete.
2998 * This will include an MI_FLUSH to get the data from GPU
2999 * to memory
3000 * clflush (obj) to invalidate the CPU cache
3001 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3002 * 5. Read/written by CPU
3003 * cache lines are loaded and dirtied
3004 * 6. Read written by GPU
3005 * Same as last GPU access
3006 *
3007 * Case 3: The constant buffer
3008 *
3009 * 1. Allocated
3010 * 2. Written by CPU
3011 * 3. Read by GPU
3012 * 4. Updated (written) by CPU again
3013 * 5. Read by GPU
3014 *
3015 * 1. Allocated
3016 * (CPU, CPU)
3017 * 2. Written by CPU
3018 * (CPU, CPU)
3019 * 3. Read by GPU
3020 * (CPU+RENDER, 0)
3021 * flush_domains = CPU
3022 * invalidate_domains = RENDER
3023 * clflush (obj)
3024 * MI_FLUSH
3025 * drm_agp_chipset_flush
3026 * 4. Updated (written) by CPU again
3027 * (CPU, CPU)
3028 * flush_domains = 0 (no previous write domain)
3029 * invalidate_domains = 0 (no new read domains)
3030 * 5. Read by GPU
3031 * (CPU+RENDER, 0)
3032 * flush_domains = CPU
3033 * invalidate_domains = RENDER
3034 * clflush (obj)
3035 * MI_FLUSH
3036 * drm_agp_chipset_flush
3037 */
c0d90829 3038static void
8b0e378a 3039i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
673a394b
EA		 3040{
3041 struct drm_device *dev = obj->dev;
9220434a 3042 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 3043 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 3044 uint32_t invalidate_domains = 0;
3045 uint32_t flush_domains = 0;
1c5d22f7 3046 uint32_t old_read_domains;
e47c68e9 3047
8b0e378a
EA		 3048 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3049 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
673a394b 3050
652c393a
JB		 3051 intel_mark_busy(dev, obj);
3052
673a394b
EA		 3053 /*
3054 * If the object isn't moving to a new write domain,
3055 * let the object stay in multiple read domains
3056 */
8b0e378a
EA		 3057 if (obj->pending_write_domain == 0)
3058 obj->pending_read_domains |= obj->read_domains;
673a394b
EA		 3059 else
3060 obj_priv->dirty = 1;
3061
3062 /*
3063 * Flush the current write domain if
3064 * the new read domains don't match. Invalidate
3065 * any read domains which differ from the old
3066 * write domain
3067 */
8b0e378a
EA		 3068 if (obj->write_domain &&
3069 obj->write_domain != obj->pending_read_domains) {
673a394b 3070 flush_domains |= obj->write_domain;
8b0e378a
EA		 3071 invalidate_domains |=
3072 obj->pending_read_domains & ~obj->write_domain;
673a394b
EA		 3073 }
3074 /*
3075 * Invalidate any read caches which may have
3076 * stale data. That is, any new read domains.
3077 */
8b0e378a 3078 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3d2a812a 3079 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
673a394b 3080 i915_gem_clflush_object(obj);
673a394b 3081
1c5d22f7
CW		 3082 old_read_domains = obj->read_domains;
3083
efbeed96
EA		 3084 /* The actual obj->write_domain will be updated with
3085 * pending_write_domain after we emit the accumulated flush for all
3086 * of our domain changes in execbuffers (which clears objects'
3087 * write_domains). So if we have a current write domain that we
3088 * aren't changing, set pending_write_domain to that.
3089 */
3090 if (flush_domains == 0 && obj->pending_write_domain == 0)
3091 obj->pending_write_domain = obj->write_domain;
8b0e378a 3092 obj->read_domains = obj->pending_read_domains;
673a394b
EA		 3093
3094 dev->invalidate_domains |= invalidate_domains;
3095 dev->flush_domains |= flush_domains;
9220434a
CW		 3096 if (obj_priv->ring)
3097 dev_priv->mm.flush_rings |= obj_priv->ring->id;
1c5d22f7
CW		 3098
3099 trace_i915_gem_object_change_domain(obj,
3100 old_read_domains,
3101 obj->write_domain);
673a394b
EA		 3102}
3103
3104/**
e47c68e9 3105 * Moves the object from a partially CPU read to a full one.
673a394b 3106 *
e47c68e9
EA		 3107 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3108 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 3109 */
e47c68e9
EA		 3110static void
3111i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
673a394b 3112{
23010e43 3113 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 3114
e47c68e9
EA		 3115 if (!obj_priv->page_cpu_valid)
3116 return;
3117
3118 /* If we're partially in the CPU read domain, finish moving it in.
3119 */
3120 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3121 int i;
3122
3123 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3124 if (obj_priv->page_cpu_valid[i])
3125 continue;
856fa198 3126 drm_clflush_pages(obj_priv->pages + i, 1);
e47c68e9 3127 }
e47c68e9
EA		 3128 }
3129
3130 /* Free the page_cpu_valid mappings which are now stale, whether
3131 * or not we've got I915_GEM_DOMAIN_CPU.
3132 */
9a298b2a 3133 kfree(obj_priv->page_cpu_valid);
e47c68e9
EA		 3134 obj_priv->page_cpu_valid = NULL;
3135}
3136
3137/**
3138 * Set the CPU read domain on a range of the object.
3139 *
3140 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3141 * not entirely valid. The page_cpu_valid member of the object flags which
3142 * pages have been flushed, and will be respected by
3143 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3144 * of the whole object.
3145 *
3146 * This function returns when the move is complete, including waiting on
3147 * flushes to occur.
3148 */
3149static int
3150i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3151 uint64_t offset, uint64_t size)
3152{
23010e43 3153 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 3154 uint32_t old_read_domains;
e47c68e9 3155 int i, ret;
673a394b 3156
e47c68e9
EA		 3157 if (offset == 0 && size == obj->size)
3158 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 3159
ba3d8d74 3160 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9 3161 if (ret != 0)
6a47baa6 3162 return ret;
e47c68e9
EA		 3163 i915_gem_object_flush_gtt_write_domain(obj);
3164
3165 /* If we're already fully in the CPU read domain, we're done. */
3166 if (obj_priv->page_cpu_valid == NULL &&
3167 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3168 return 0;
673a394b 3169
e47c68e9
EA		 3170 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3171 * newly adding I915_GEM_DOMAIN_CPU
3172 */
673a394b 3173 if (obj_priv->page_cpu_valid == NULL) {
9a298b2a
EA		 3174 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3175 GFP_KERNEL);
e47c68e9
EA		 3176 if (obj_priv->page_cpu_valid == NULL)
3177 return -ENOMEM;
3178 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3179 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
673a394b
EA		 3180
3181 /* Flush the cache on any pages that are still invalid from the CPU's
3182 * perspective.
3183 */
e47c68e9
EA		 3184 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3185 i++) {
673a394b
EA		 3186 if (obj_priv->page_cpu_valid[i])
3187 continue;
3188
856fa198 3189 drm_clflush_pages(obj_priv->pages + i, 1);
673a394b
EA		 3190
3191 obj_priv->page_cpu_valid[i] = 1;
3192 }
3193
e47c68e9
EA		 3194 /* It should now be out of any other write domains, and we can update
3195 * the domain values for our changes.
3196 */
3197 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3198
1c5d22f7 3199 old_read_domains = obj->read_domains;
e47c68e9
EA		 3200 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3201
1c5d22f7
CW		 3202 trace_i915_gem_object_change_domain(obj,
3203 old_read_domains,
3204 obj->write_domain);
3205
673a394b
EA		 3206 return 0;
3207}
3208
673a394b
EA		 3209/**
3210 * Pin an object to the GTT and evaluate the relocations landing in it.
3211 */
3212static int
3213i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3214 struct drm_file *file_priv,
76446cac 3215 struct drm_i915_gem_exec_object2 *entry,
40a5f0de 3216 struct drm_i915_gem_relocation_entry *relocs)
673a394b
EA		 3217{
3218 struct drm_device *dev = obj->dev;
0839ccb8 3219 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 3220 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 3221 int i, ret;
0839ccb8 3222 void __iomem *reloc_page;
76446cac
JB		 3223 bool need_fence;
3224
3225 need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3226 obj_priv->tiling_mode != I915_TILING_NONE;
3227
3228 /* Check fence reg constraints and rebind if necessary */
808b24d6
CW		 3229 if (need_fence &&
3230 !i915_gem_object_fence_offset_ok(obj,
3231 obj_priv->tiling_mode)) {
3232 ret = i915_gem_object_unbind(obj);
3233 if (ret)
3234 return ret;
3235 }
673a394b
EA		 3236
3237 /* Choose the GTT offset for our buffer and put it there. */
3238 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3239 if (ret)
3240 return ret;
3241
76446cac
JB		 3242 /*
3243 * Pre-965 chips need a fence register set up in order to
3244 * properly handle blits to/from tiled surfaces.
3245 */
3246 if (need_fence) {
53640e1d 3247 ret = i915_gem_object_get_fence_reg(obj, true);
76446cac 3248 if (ret != 0) {
76446cac
JB		 3249 i915_gem_object_unpin(obj);
3250 return ret;
3251 }
53640e1d
CW		 3252
3253 dev_priv->fence_regs[obj_priv->fence_reg].gpu = true;
76446cac
JB		 3254 }
3255
673a394b
EA		 3256 entry->offset = obj_priv->gtt_offset;
3257
673a394b
EA		 3258 /* Apply the relocations, using the GTT aperture to avoid cache
3259 * flushing requirements.
3260 */
3261 for (i = 0; i < entry->relocation_count; i++) {
40a5f0de 3262 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
673a394b
EA		 3263 struct drm_gem_object *target_obj;
3264 struct drm_i915_gem_object *target_obj_priv;
3043c60c
EA		 3265 uint32_t reloc_val, reloc_offset;
3266 uint32_t __iomem *reloc_entry;
673a394b 3267
673a394b 3268 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
40a5f0de 3269 reloc->target_handle);
673a394b
EA		 3270 if (target_obj == NULL) {
3271 i915_gem_object_unpin(obj);
bf79cb91 3272 return -ENOENT;
673a394b 3273 }
23010e43 3274 target_obj_priv = to_intel_bo(target_obj);
673a394b 3275
8542a0bb
CW		 3276#if WATCH_RELOC
3277 DRM_INFO("%s: obj %p offset %08x target %d "
3278 "read %08x write %08x gtt %08x "
3279 "presumed %08x delta %08x\n",
3280 __func__,
3281 obj,
3282 (int) reloc->offset,
3283 (int) reloc->target_handle,
3284 (int) reloc->read_domains,
3285 (int) reloc->write_domain,
3286 (int) target_obj_priv->gtt_offset,
3287 (int) reloc->presumed_offset,
3288 reloc->delta);
3289#endif
3290
673a394b
EA		 3291 /* The target buffer should have appeared before us in the
3292 * exec_object list, so it should have a GTT space bound by now.
3293 */
3294 if (target_obj_priv->gtt_space == NULL) {
3295 DRM_ERROR("No GTT space found for object %d\n",
40a5f0de 3296 reloc->target_handle);
673a394b
EA		 3297 drm_gem_object_unreference(target_obj);
3298 i915_gem_object_unpin(obj);
3299 return -EINVAL;
3300 }
3301
8542a0bb 3302 /* Validate that the target is in a valid r/w GPU domain */
16edd550
DV		 3303 if (reloc->write_domain & (reloc->write_domain - 1)) {
3304 DRM_ERROR("reloc with multiple write domains: "
3305 "obj %p target %d offset %d "
3306 "read %08x write %08x",
3307 obj, reloc->target_handle,
3308 (int) reloc->offset,
3309 reloc->read_domains,
3310 reloc->write_domain);
3311 return -EINVAL;
3312 }
40a5f0de
EA		 3313 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3314 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 3315 DRM_ERROR("reloc with read/write CPU domains: "
3316 "obj %p target %d offset %d "
3317 "read %08x write %08x",
40a5f0de
EA		 3318 obj, reloc->target_handle,
3319 (int) reloc->offset,
3320 reloc->read_domains,
3321 reloc->write_domain);
491152b8
CW		 3322 drm_gem_object_unreference(target_obj);
3323 i915_gem_object_unpin(obj);
e47c68e9
EA		 3324 return -EINVAL;
3325 }
40a5f0de
EA		 3326 if (reloc->write_domain && target_obj->pending_write_domain &&
3327 reloc->write_domain != target_obj->pending_write_domain) {
673a394b
EA		 3328 DRM_ERROR("Write domain conflict: "
3329 "obj %p target %d offset %d "
3330 "new %08x old %08x\n",
40a5f0de
EA		 3331 obj, reloc->target_handle,
3332 (int) reloc->offset,
3333 reloc->write_domain,
673a394b
EA		 3334 target_obj->pending_write_domain);
3335 drm_gem_object_unreference(target_obj);
3336 i915_gem_object_unpin(obj);
3337 return -EINVAL;
3338 }
3339
40a5f0de
EA		 3340 target_obj->pending_read_domains |= reloc->read_domains;
3341 target_obj->pending_write_domain |= reloc->write_domain;
673a394b
EA		 3342
3343 /* If the relocation already has the right value in it, no
3344 * more work needs to be done.
3345 */
40a5f0de 3346 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
673a394b
EA		 3347 drm_gem_object_unreference(target_obj);
3348 continue;
3349 }
3350
8542a0bb
CW		 3351 /* Check that the relocation address is valid... */
3352 if (reloc->offset > obj->size - 4) {
3353 DRM_ERROR("Relocation beyond object bounds: "
3354 "obj %p target %d offset %d size %d.\n",
3355 obj, reloc->target_handle,
3356 (int) reloc->offset, (int) obj->size);
3357 drm_gem_object_unreference(target_obj);
3358 i915_gem_object_unpin(obj);
3359 return -EINVAL;
3360 }
3361 if (reloc->offset & 3) {
3362 DRM_ERROR("Relocation not 4-byte aligned: "
3363 "obj %p target %d offset %d.\n",
3364 obj, reloc->target_handle,
3365 (int) reloc->offset);
3366 drm_gem_object_unreference(target_obj);
3367 i915_gem_object_unpin(obj);
3368 return -EINVAL;
3369 }
3370
3371 /* and points to somewhere within the target object. */
3372 if (reloc->delta >= target_obj->size) {
3373 DRM_ERROR("Relocation beyond target object bounds: "
3374 "obj %p target %d delta %d size %d.\n",
3375 obj, reloc->target_handle,
3376 (int) reloc->delta, (int) target_obj->size);
3377 drm_gem_object_unreference(target_obj);
3378 i915_gem_object_unpin(obj);
3379 return -EINVAL;
3380 }
3381
2ef7eeaa
EA		 3382 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3383 if (ret != 0) {
3384 drm_gem_object_unreference(target_obj);
3385 i915_gem_object_unpin(obj);
3386 return -EINVAL;
673a394b
EA		 3387 }
3388
3389 /* Map the page containing the relocation we're going to
3390 * perform.
3391 */
40a5f0de 3392 reloc_offset = obj_priv->gtt_offset + reloc->offset;
0839ccb8
KP		 3393 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3394 (reloc_offset &
fca3ec01
CW		 3395 ~(PAGE_SIZE - 1)),
3396 KM_USER0);
3043c60c 3397 reloc_entry = (uint32_t __iomem *)(reloc_page +
0839ccb8 3398 (reloc_offset & (PAGE_SIZE - 1)));
40a5f0de 3399 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
673a394b 3400
673a394b 3401 writel(reloc_val, reloc_entry);
fca3ec01 3402 io_mapping_unmap_atomic(reloc_page, KM_USER0);
673a394b 3403
40a5f0de
EA		 3404 /* The updated presumed offset for this entry will be
3405 * copied back out to the user.
673a394b 3406 */
40a5f0de 3407 reloc->presumed_offset = target_obj_priv->gtt_offset;
673a394b
EA		 3408
3409 drm_gem_object_unreference(target_obj);
3410 }
3411
673a394b
EA		 3412 return 0;
3413}
3414
673a394b
EA		 3415/* Throttle our rendering by waiting until the ring has completed our requests
3416 * emitted over 20 msec ago.
3417 *
b962442e
EA		 3418 * Note that if we were to use the current jiffies each time around the loop,
3419 * we wouldn't escape the function with any frames outstanding if the time to
3420 * render a frame was over 20ms.
3421 *
673a394b
EA		 3422 * This should get us reasonable parallelism between CPU and GPU but also
3423 * relatively low latency when blocking on a particular request to finish.
3424 */
3425static int
f787a5f5 3426i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
673a394b 3427{
f787a5f5
CW		 3428 struct drm_i915_private *dev_priv = dev->dev_private;
3429 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e 3430 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
f787a5f5
CW		 3431 struct drm_i915_gem_request *request;
3432 struct intel_ring_buffer *ring = NULL;
3433 u32 seqno = 0;
3434 int ret;
673a394b 3435
1c25595f 3436 spin_lock(&file_priv->mm.lock);
f787a5f5 3437 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
b962442e
EA		 3438 if (time_after_eq(request->emitted_jiffies, recent_enough))
3439 break;
3440
f787a5f5
CW		 3441 ring = request->ring;
3442 seqno = request->seqno;
b962442e 3443 }
1c25595f 3444 spin_unlock(&file_priv->mm.lock);
f787a5f5
CW		 3445
3446 if (seqno == 0)
3447 return 0;
3448
3449 ret = 0;
3450 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
3451 /* And wait for the seqno passing without holding any locks and
3452 * causing extra latency for others. This is safe as the irq
3453 * generation is designed to be run atomically and so is
3454 * lockless.
3455 */
3456 ring->user_irq_get(dev, ring);
3457 ret = wait_event_interruptible(ring->irq_queue,
3458 i915_seqno_passed(ring->get_seqno(dev, ring), seqno)
3459 || atomic_read(&dev_priv->mm.wedged));
3460 ring->user_irq_put(dev, ring);
3461
3462 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3463 ret = -EIO;
3464 }
3465
3466 if (ret == 0)
3467 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
b962442e 3468
673a394b
EA		 3469 return ret;
3470}
3471
40a5f0de 3472static int
76446cac 3473i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
40a5f0de
EA		 3474 uint32_t buffer_count,
3475 struct drm_i915_gem_relocation_entry **relocs)
3476{
3477 uint32_t reloc_count = 0, reloc_index = 0, i;
3478 int ret;
3479
3480 *relocs = NULL;
3481 for (i = 0; i < buffer_count; i++) {
3482 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3483 return -EINVAL;
3484 reloc_count += exec_list[i].relocation_count;
3485 }
3486
8e7d2b2c 3487 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
76446cac
JB		 3488 if (*relocs == NULL) {
3489 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
40a5f0de 3490 return -ENOMEM;
76446cac 3491 }
40a5f0de
EA		 3492
3493 for (i = 0; i < buffer_count; i++) {
3494 struct drm_i915_gem_relocation_entry __user *user_relocs;
3495
3496 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3497
3498 ret = copy_from_user(&(*relocs)[reloc_index],
3499 user_relocs,
3500 exec_list[i].relocation_count *
3501 sizeof(**relocs));
3502 if (ret != 0) {
8e7d2b2c 3503 drm_free_large(*relocs);
40a5f0de 3504 *relocs = NULL;
2bc43b5c 3505 return -EFAULT;
40a5f0de
EA		 3506 }
3507
3508 reloc_index += exec_list[i].relocation_count;
3509 }
3510
2bc43b5c 3511 return 0;
40a5f0de
EA		 3512}
3513
3514static int
76446cac 3515i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
40a5f0de
EA		 3516 uint32_t buffer_count,
3517 struct drm_i915_gem_relocation_entry *relocs)
3518{
3519 uint32_t reloc_count = 0, i;
2bc43b5c 3520 int ret = 0;
40a5f0de 3521
93533c29
CW		 3522 if (relocs == NULL)
3523 return 0;
3524
40a5f0de
EA		 3525 for (i = 0; i < buffer_count; i++) {
3526 struct drm_i915_gem_relocation_entry __user *user_relocs;
2bc43b5c 3527 int unwritten;
40a5f0de
EA		 3528
3529 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3530
2bc43b5c
FM		 3531 unwritten = copy_to_user(user_relocs,
3532 &relocs[reloc_count],
3533 exec_list[i].relocation_count *
3534 sizeof(*relocs));
3535
3536 if (unwritten) {
3537 ret = -EFAULT;
3538 goto err;
40a5f0de
EA		 3539 }
3540
3541 reloc_count += exec_list[i].relocation_count;
3542 }
3543
2bc43b5c 3544err:
8e7d2b2c 3545 drm_free_large(relocs);
40a5f0de
EA		 3546
3547 return ret;
3548}
3549
83d60795 3550static int
76446cac 3551i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
83d60795
CW		 3552 uint64_t exec_offset)
3553{
3554 uint32_t exec_start, exec_len;
3555
3556 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3557 exec_len = (uint32_t) exec->batch_len;
3558
3559 if ((exec_start | exec_len) & 0x7)
3560 return -EINVAL;
3561
3562 if (!exec_start)
3563 return -EINVAL;
3564
3565 return 0;
3566}
3567
e6c3a2a6 3568static int
6b95a207
KH		 3569i915_gem_wait_for_pending_flip(struct drm_device *dev,
3570 struct drm_gem_object **object_list,
3571 int count)
3572{
3573 drm_i915_private_t *dev_priv = dev->dev_private;
3574 struct drm_i915_gem_object *obj_priv;
3575 DEFINE_WAIT(wait);
3576 int i, ret = 0;
3577
3578 for (;;) {
3579 prepare_to_wait(&dev_priv->pending_flip_queue,
3580 &wait, TASK_INTERRUPTIBLE);
3581 for (i = 0; i < count; i++) {
23010e43 3582 obj_priv = to_intel_bo(object_list[i]);
6b95a207
KH		 3583 if (atomic_read(&obj_priv->pending_flip) > 0)
3584 break;
3585 }
3586 if (i == count)
3587 break;
3588
3589 if (!signal_pending(current)) {
3590 mutex_unlock(&dev->struct_mutex);
3591 schedule();
3592 mutex_lock(&dev->struct_mutex);
3593 continue;
3594 }
3595 ret = -ERESTARTSYS;
3596 break;
3597 }
3598 finish_wait(&dev_priv->pending_flip_queue, &wait);
3599
3600 return ret;
3601}
3602
8dc5d147 3603static int
76446cac
JB		 3604i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3605 struct drm_file *file_priv,
3606 struct drm_i915_gem_execbuffer2 *args,
3607 struct drm_i915_gem_exec_object2 *exec_list)
673a394b
EA		 3608{
3609 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3610 struct drm_gem_object **object_list = NULL;
3611 struct drm_gem_object *batch_obj;
b70d11da 3612 struct drm_i915_gem_object *obj_priv;
201361a5 3613 struct drm_clip_rect *cliprects = NULL;
93533c29 3614 struct drm_i915_gem_relocation_entry *relocs = NULL;
8dc5d147 3615 struct drm_i915_gem_request *request = NULL;
30dbf0c0 3616 int ret, ret2, i, pinned = 0;
673a394b 3617 uint64_t exec_offset;
5c12a07e 3618 uint32_t reloc_index;
6b95a207 3619 int pin_tries, flips;
673a394b 3620
852835f3
ZN		 3621 struct intel_ring_buffer *ring = NULL;
3622
30dbf0c0
CW		 3623 ret = i915_gem_check_is_wedged(dev);
3624 if (ret)
3625 return ret;
3626
673a394b
EA		 3627#if WATCH_EXEC
3628 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3629 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3630#endif
d1b851fc
ZN		 3631 if (args->flags & I915_EXEC_BSD) {
3632 if (!HAS_BSD(dev)) {
3633 DRM_ERROR("execbuf with wrong flag\n");
3634 return -EINVAL;
3635 }
3636 ring = &dev_priv->bsd_ring;
3637 } else {
3638 ring = &dev_priv->render_ring;
3639 }
3640
4f481ed2
EA		 3641 if (args->buffer_count < 1) {
3642 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3643 return -EINVAL;
3644 }
c8e0f93a 3645 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
76446cac
JB		 3646 if (object_list == NULL) {
3647 DRM_ERROR("Failed to allocate object list for %d buffers\n",
673a394b
EA		 3648 args->buffer_count);
3649 ret = -ENOMEM;
3650 goto pre_mutex_err;
3651 }
673a394b 3652
201361a5 3653 if (args->num_cliprects != 0) {
9a298b2a
EA		 3654 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3655 GFP_KERNEL);
a40e8d31
OA		 3656 if (cliprects == NULL) {
3657 ret = -ENOMEM;
201361a5 3658 goto pre_mutex_err;
a40e8d31 3659 }
201361a5
EA		 3660
3661 ret = copy_from_user(cliprects,
3662 (struct drm_clip_rect __user *)
3663 (uintptr_t) args->cliprects_ptr,
3664 sizeof(*cliprects) * args->num_cliprects);
3665 if (ret != 0) {
3666 DRM_ERROR("copy %d cliprects failed: %d\n",
3667 args->num_cliprects, ret);
c877cdce 3668 ret = -EFAULT;
201361a5
EA		 3669 goto pre_mutex_err;
3670 }
3671 }
3672
8dc5d147
CW		 3673 request = kzalloc(sizeof(*request), GFP_KERNEL);
3674 if (request == NULL) {
3675 ret = -ENOMEM;
3676 goto pre_mutex_err;
3677 }
3678
40a5f0de
EA		 3679 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3680 &relocs);
3681 if (ret != 0)
3682 goto pre_mutex_err;
3683
76c1dec1
CW		 3684 ret = i915_mutex_lock_interruptible(dev);
3685 if (ret)
3686 goto pre_mutex_err;
673a394b
EA		 3687
3688 i915_verify_inactive(dev, __FILE__, __LINE__);
3689
673a394b 3690 if (dev_priv->mm.suspended) {
673a394b 3691 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3692 ret = -EBUSY;
3693 goto pre_mutex_err;
673a394b
EA		 3694 }
3695
ac94a962 3696 /* Look up object handles */
6b95a207 3697 flips = 0;
673a394b
EA		 3698 for (i = 0; i < args->buffer_count; i++) {
3699 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3700 exec_list[i].handle);
3701 if (object_list[i] == NULL) {
3702 DRM_ERROR("Invalid object handle %d at index %d\n",
3703 exec_list[i].handle, i);
0ce907f8
CW		 3704 /* prevent error path from reading uninitialized data */
3705 args->buffer_count = i + 1;
bf79cb91 3706 ret = -ENOENT;
673a394b
EA		 3707 goto err;
3708 }
b70d11da 3709
23010e43 3710 obj_priv = to_intel_bo(object_list[i]);
b70d11da
KH		 3711 if (obj_priv->in_execbuffer) {
3712 DRM_ERROR("Object %p appears more than once in object list\n",
3713 object_list[i]);
0ce907f8
CW		 3714 /* prevent error path from reading uninitialized data */
3715 args->buffer_count = i + 1;
bf79cb91 3716 ret = -EINVAL;
b70d11da
KH		 3717 goto err;
3718 }
3719 obj_priv->in_execbuffer = true;
6b95a207
KH		 3720 flips += atomic_read(&obj_priv->pending_flip);
3721 }
3722
3723 if (flips > 0) {
3724 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3725 args->buffer_count);
3726 if (ret)
3727 goto err;
ac94a962 3728 }
673a394b 3729
ac94a962
KP		 3730 /* Pin and relocate */
3731 for (pin_tries = 0; ; pin_tries++) {
3732 ret = 0;
40a5f0de
EA		 3733 reloc_index = 0;
3734
ac94a962
KP		 3735 for (i = 0; i < args->buffer_count; i++) {
3736 object_list[i]->pending_read_domains = 0;
3737 object_list[i]->pending_write_domain = 0;
3738 ret = i915_gem_object_pin_and_relocate(object_list[i],
3739 file_priv,
40a5f0de
EA		 3740 &exec_list[i],
3741 &relocs[reloc_index]);
ac94a962
KP		 3742 if (ret)
3743 break;
3744 pinned = i + 1;
40a5f0de 3745 reloc_index += exec_list[i].relocation_count;
ac94a962
KP		 3746 }
3747 /* success */
3748 if (ret == 0)
3749 break;
3750
3751 /* error other than GTT full, or we've already tried again */
2939e1f5 3752 if (ret != -ENOSPC || pin_tries >= 1) {
07f73f69
CW		 3753 if (ret != -ERESTARTSYS) {
3754 unsigned long long total_size = 0;
3d1cc470
CW		 3755 int num_fences = 0;
3756 for (i = 0; i < args->buffer_count; i++) {
43b27f40 3757 obj_priv = to_intel_bo(object_list[i]);
3d1cc470 3758
07f73f69 3759 total_size += object_list[i]->size;
3d1cc470
CW		 3760 num_fences +=
3761 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3762 obj_priv->tiling_mode != I915_TILING_NONE;
3763 }
3764 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
07f73f69 3765 pinned+1, args->buffer_count,
3d1cc470
CW		 3766 total_size, num_fences,
3767 ret);
07f73f69
CW		 3768 DRM_ERROR("%d objects [%d pinned], "
3769 "%d object bytes [%d pinned], "
3770 "%d/%d gtt bytes\n",
3771 atomic_read(&dev->object_count),
3772 atomic_read(&dev->pin_count),
3773 atomic_read(&dev->object_memory),
3774 atomic_read(&dev->pin_memory),
3775 atomic_read(&dev->gtt_memory),
3776 dev->gtt_total);
3777 }
673a394b
EA		 3778 goto err;
3779 }
ac94a962
KP		 3780
3781 /* unpin all of our buffers */
3782 for (i = 0; i < pinned; i++)
3783 i915_gem_object_unpin(object_list[i]);
b1177636 3784 pinned = 0;
ac94a962
KP		 3785
3786 /* evict everyone we can from the aperture */
3787 ret = i915_gem_evict_everything(dev);
07f73f69 3788 if (ret && ret != -ENOSPC)
ac94a962 3789 goto err;
673a394b
EA		 3790 }
3791
3792 /* Set the pending read domains for the batch buffer to COMMAND */
3793 batch_obj = object_list[args->buffer_count-1];
5f26a2c7
CW		 3794 if (batch_obj->pending_write_domain) {
3795 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3796 ret = -EINVAL;
3797 goto err;
3798 }
3799 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
673a394b 3800
83d60795
CW		 3801 /* Sanity check the batch buffer, prior to moving objects */
3802 exec_offset = exec_list[args->buffer_count - 1].offset;
3803 ret = i915_gem_check_execbuffer (args, exec_offset);
3804 if (ret != 0) {
3805 DRM_ERROR("execbuf with invalid offset/length\n");
3806 goto err;
3807 }
3808
673a394b
EA		 3809 i915_verify_inactive(dev, __FILE__, __LINE__);
3810
646f0f6e
KP		 3811 /* Zero the global flush/invalidate flags. These
3812 * will be modified as new domains are computed
3813 * for each object
3814 */
3815 dev->invalidate_domains = 0;
3816 dev->flush_domains = 0;
9220434a 3817 dev_priv->mm.flush_rings = 0;
646f0f6e 3818
673a394b
EA		 3819 for (i = 0; i < args->buffer_count; i++) {
3820 struct drm_gem_object *obj = object_list[i];
673a394b 3821
646f0f6e 3822 /* Compute new gpu domains and update invalidate/flush */
8b0e378a 3823 i915_gem_object_set_to_gpu_domain(obj);
673a394b
EA		 3824 }
3825
3826 i915_verify_inactive(dev, __FILE__, __LINE__);
3827
646f0f6e
KP		 3828 if (dev->invalidate_domains | dev->flush_domains) {
3829#if WATCH_EXEC
3830 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3831 __func__,
3832 dev->invalidate_domains,
3833 dev->flush_domains);
3834#endif
c78ec30b 3835 i915_gem_flush(dev, file_priv,
646f0f6e 3836 dev->invalidate_domains,
9220434a
CW		 3837 dev->flush_domains,
3838 dev_priv->mm.flush_rings);
a6910434
DV		 3839 }
3840
efbeed96
EA		 3841 for (i = 0; i < args->buffer_count; i++) {
3842 struct drm_gem_object *obj = object_list[i];
23010e43 3843 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 3844 uint32_t old_write_domain = obj->write_domain;
efbeed96
EA		 3845
3846 obj->write_domain = obj->pending_write_domain;
99fcb766
DV		 3847 if (obj->write_domain)
3848 list_move_tail(&obj_priv->gpu_write_list,
3849 &dev_priv->mm.gpu_write_list);
3850 else
3851 list_del_init(&obj_priv->gpu_write_list);
3852
1c5d22f7
CW		 3853 trace_i915_gem_object_change_domain(obj,
3854 obj->read_domains,
3855 old_write_domain);
efbeed96
EA		 3856 }
3857
673a394b
EA		 3858 i915_verify_inactive(dev, __FILE__, __LINE__);
3859
3860#if WATCH_COHERENCY
3861 for (i = 0; i < args->buffer_count; i++) {
3862 i915_gem_object_check_coherency(object_list[i],
3863 exec_list[i].handle);
3864 }
3865#endif
3866
673a394b 3867#if WATCH_EXEC
6911a9b8 3868 i915_gem_dump_object(batch_obj,
673a394b
EA		 3869 args->batch_len,
3870 __func__,
3871 ~0);
3872#endif
3873
673a394b 3874 /* Exec the batchbuffer */
852835f3
ZN		 3875 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3876 cliprects, exec_offset);
673a394b
EA		 3877 if (ret) {
3878 DRM_ERROR("dispatch failed %d\n", ret);
3879 goto err;
3880 }
3881
3882 /*
3883 * Ensure that the commands in the batch buffer are
3884 * finished before the interrupt fires
3885 */
8a1a49f9 3886 i915_retire_commands(dev, ring);
673a394b
EA		 3887
3888 i915_verify_inactive(dev, __FILE__, __LINE__);
3889
617dbe27
DV		 3890 for (i = 0; i < args->buffer_count; i++) {
3891 struct drm_gem_object *obj = object_list[i];
3892 obj_priv = to_intel_bo(obj);
3893
3894 i915_gem_object_move_to_active(obj, ring);
617dbe27 3895 }
a56ba56c 3896
5c12a07e 3897 i915_add_request(dev, file_priv, request, ring);
8dc5d147 3898 request = NULL;
673a394b 3899
673a394b
EA		 3900 i915_verify_inactive(dev, __FILE__, __LINE__);
3901
673a394b 3902err:
aad87dff
JL		 3903 for (i = 0; i < pinned; i++)
3904 i915_gem_object_unpin(object_list[i]);
3905
b70d11da
KH		 3906 for (i = 0; i < args->buffer_count; i++) {
3907 if (object_list[i]) {
23010e43 3908 obj_priv = to_intel_bo(object_list[i]);
b70d11da
KH		 3909 obj_priv->in_execbuffer = false;
3910 }
aad87dff 3911 drm_gem_object_unreference(object_list[i]);
b70d11da 3912 }
673a394b 3913
673a394b
EA		 3914 mutex_unlock(&dev->struct_mutex);
3915
93533c29 3916pre_mutex_err:
40a5f0de
EA		 3917 /* Copy the updated relocations out regardless of current error
3918 * state. Failure to update the relocs would mean that the next
3919 * time userland calls execbuf, it would do so with presumed offset
3920 * state that didn't match the actual object state.
3921 */
3922 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3923 relocs);
3924 if (ret2 != 0) {
3925 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3926
3927 if (ret == 0)
3928 ret = ret2;
3929 }
3930
8e7d2b2c 3931 drm_free_large(object_list);
9a298b2a 3932 kfree(cliprects);
8dc5d147 3933 kfree(request);
673a394b
EA		 3934
3935 return ret;
3936}
3937
76446cac
JB		 3938/*
3939 * Legacy execbuffer just creates an exec2 list from the original exec object
3940 * list array and passes it to the real function.
3941 */
3942int
3943i915_gem_execbuffer(struct drm_device *dev, void *data,
3944 struct drm_file *file_priv)
3945{
3946 struct drm_i915_gem_execbuffer *args = data;
3947 struct drm_i915_gem_execbuffer2 exec2;
3948 struct drm_i915_gem_exec_object *exec_list = NULL;
3949 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3950 int ret, i;
3951
3952#if WATCH_EXEC
3953 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3954 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3955#endif
3956
3957 if (args->buffer_count < 1) {
3958 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3959 return -EINVAL;
3960 }
3961
3962 /* Copy in the exec list from userland */
3963 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3964 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3965 if (exec_list == NULL || exec2_list == NULL) {
3966 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3967 args->buffer_count);
3968 drm_free_large(exec_list);
3969 drm_free_large(exec2_list);
3970 return -ENOMEM;
3971 }
3972 ret = copy_from_user(exec_list,
3973 (struct drm_i915_relocation_entry __user *)
3974 (uintptr_t) args->buffers_ptr,
3975 sizeof(*exec_list) * args->buffer_count);
3976 if (ret != 0) {
3977 DRM_ERROR("copy %d exec entries failed %d\n",
3978 args->buffer_count, ret);
3979 drm_free_large(exec_list);
3980 drm_free_large(exec2_list);
3981 return -EFAULT;
3982 }
3983
3984 for (i = 0; i < args->buffer_count; i++) {
3985 exec2_list[i].handle = exec_list[i].handle;
3986 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3987 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3988 exec2_list[i].alignment = exec_list[i].alignment;
3989 exec2_list[i].offset = exec_list[i].offset;
a6c45cf0 3990 if (INTEL_INFO(dev)->gen < 4)
76446cac
JB		 3991 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3992 else
3993 exec2_list[i].flags = 0;
3994 }
3995
3996 exec2.buffers_ptr = args->buffers_ptr;
3997 exec2.buffer_count = args->buffer_count;
3998 exec2.batch_start_offset = args->batch_start_offset;
3999 exec2.batch_len = args->batch_len;
4000 exec2.DR1 = args->DR1;
4001 exec2.DR4 = args->DR4;
4002 exec2.num_cliprects = args->num_cliprects;
4003 exec2.cliprects_ptr = args->cliprects_ptr;
852835f3 4004 exec2.flags = I915_EXEC_RENDER;
76446cac
JB		 4005
4006 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4007 if (!ret) {
4008 /* Copy the new buffer offsets back to the user's exec list. */
4009 for (i = 0; i < args->buffer_count; i++)
4010 exec_list[i].offset = exec2_list[i].offset;
4011 /* ... and back out to userspace */
4012 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4013 (uintptr_t) args->buffers_ptr,
4014 exec_list,
4015 sizeof(*exec_list) * args->buffer_count);
4016 if (ret) {
4017 ret = -EFAULT;
4018 DRM_ERROR("failed to copy %d exec entries "
4019 "back to user (%d)\n",
4020 args->buffer_count, ret);
4021 }
76446cac
JB		 4022 }
4023
4024 drm_free_large(exec_list);
4025 drm_free_large(exec2_list);
4026 return ret;
4027}
4028
4029int
4030i915_gem_execbuffer2(struct drm_device *dev, void *data,
4031 struct drm_file *file_priv)
4032{
4033 struct drm_i915_gem_execbuffer2 *args = data;
4034 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4035 int ret;
4036
4037#if WATCH_EXEC
4038 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4039 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4040#endif
4041
4042 if (args->buffer_count < 1) {
4043 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4044 return -EINVAL;
4045 }
4046
4047 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4048 if (exec2_list == NULL) {
4049 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4050 args->buffer_count);
4051 return -ENOMEM;
4052 }
4053 ret = copy_from_user(exec2_list,
4054 (struct drm_i915_relocation_entry __user *)
4055 (uintptr_t) args->buffers_ptr,
4056 sizeof(*exec2_list) * args->buffer_count);
4057 if (ret != 0) {
4058 DRM_ERROR("copy %d exec entries failed %d\n",
4059 args->buffer_count, ret);
4060 drm_free_large(exec2_list);
4061 return -EFAULT;
4062 }
4063
4064 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4065 if (!ret) {
4066 /* Copy the new buffer offsets back to the user's exec list. */
4067 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4068 (uintptr_t) args->buffers_ptr,
4069 exec2_list,
4070 sizeof(*exec2_list) * args->buffer_count);
4071 if (ret) {
4072 ret = -EFAULT;
4073 DRM_ERROR("failed to copy %d exec entries "
4074 "back to user (%d)\n",
4075 args->buffer_count, ret);
4076 }
4077 }
4078
4079 drm_free_large(exec2_list);
4080 return ret;
4081}
4082
673a394b
EA		 4083int
4084i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4085{
4086 struct drm_device *dev = obj->dev;
f13d3f73 4087 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 4088 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 4089 int ret;
4090
778c3544
DV		 4091 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4092
673a394b 4093 i915_verify_inactive(dev, __FILE__, __LINE__);
ac0c6b5a
CW		 4094
4095 if (obj_priv->gtt_space != NULL) {
4096 if (alignment == 0)
4097 alignment = i915_gem_get_gtt_alignment(obj);
4098 if (obj_priv->gtt_offset & (alignment - 1)) {
ae7d49d8
CW		 4099 WARN(obj_priv->pin_count,
4100 "bo is already pinned with incorrect alignment:"
4101 " offset=%x, req.alignment=%x\n",
4102 obj_priv->gtt_offset, alignment);
ac0c6b5a
CW		 4103 ret = i915_gem_object_unbind(obj);
4104 if (ret)
4105 return ret;
4106 }
4107 }
4108
673a394b
EA		 4109 if (obj_priv->gtt_space == NULL) {
4110 ret = i915_gem_object_bind_to_gtt(obj, alignment);
9731129c 4111 if (ret)
673a394b 4112 return ret;
22c344e9 4113 }
76446cac 4114
673a394b
EA		 4115 obj_priv->pin_count++;
4116
4117 /* If the object is not active and not pending a flush,
4118 * remove it from the inactive list
4119 */
4120 if (obj_priv->pin_count == 1) {
4121 atomic_inc(&dev->pin_count);
4122 atomic_add(obj->size, &dev->pin_memory);
f13d3f73
CW		 4123 if (!obj_priv->active)
4124 list_move_tail(&obj_priv->list,
4125 &dev_priv->mm.pinned_list);
673a394b
EA		 4126 }
4127 i915_verify_inactive(dev, __FILE__, __LINE__);
4128
4129 return 0;
4130}
4131
4132void
4133i915_gem_object_unpin(struct drm_gem_object *obj)
4134{
4135 struct drm_device *dev = obj->dev;
4136 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 4137 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 4138
4139 i915_verify_inactive(dev, __FILE__, __LINE__);
4140 obj_priv->pin_count--;
4141 BUG_ON(obj_priv->pin_count < 0);
4142 BUG_ON(obj_priv->gtt_space == NULL);
4143
4144 /* If the object is no longer pinned, and is
4145 * neither active nor being flushed, then stick it on
4146 * the inactive list
4147 */
4148 if (obj_priv->pin_count == 0) {
f13d3f73 4149 if (!obj_priv->active)
673a394b
EA		 4150 list_move_tail(&obj_priv->list,
4151 &dev_priv->mm.inactive_list);
4152 atomic_dec(&dev->pin_count);
4153 atomic_sub(obj->size, &dev->pin_memory);
4154 }
4155 i915_verify_inactive(dev, __FILE__, __LINE__);
4156}
4157
4158int
4159i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4160 struct drm_file *file_priv)
4161{
4162 struct drm_i915_gem_pin *args = data;
4163 struct drm_gem_object *obj;
4164 struct drm_i915_gem_object *obj_priv;
4165 int ret;
4166
673a394b
EA		 4167 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4168 if (obj == NULL) {
4169 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4170 args->handle);
bf79cb91 4171 return -ENOENT;
673a394b 4172 }
23010e43 4173 obj_priv = to_intel_bo(obj);
673a394b 4174
76c1dec1
CW		 4175 ret = i915_mutex_lock_interruptible(dev);
4176 if (ret) {
4177 drm_gem_object_unreference_unlocked(obj);
4178 return ret;
4179 }
4180
bb6baf76
CW		 4181 if (obj_priv->madv != I915_MADV_WILLNEED) {
4182 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3ef94daa
CW		 4183 drm_gem_object_unreference(obj);
4184 mutex_unlock(&dev->struct_mutex);
4185 return -EINVAL;
4186 }
4187
79e53945
JB		 4188 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4189 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4190 args->handle);
96dec61d 4191 drm_gem_object_unreference(obj);
673a394b 4192 mutex_unlock(&dev->struct_mutex);
79e53945
JB		 4193 return -EINVAL;
4194 }
4195
4196 obj_priv->user_pin_count++;
4197 obj_priv->pin_filp = file_priv;
4198 if (obj_priv->user_pin_count == 1) {
4199 ret = i915_gem_object_pin(obj, args->alignment);
4200 if (ret != 0) {
4201 drm_gem_object_unreference(obj);
4202 mutex_unlock(&dev->struct_mutex);
4203 return ret;
4204 }
673a394b
EA		 4205 }
4206
4207 /* XXX - flush the CPU caches for pinned objects
4208 * as the X server doesn't manage domains yet
4209 */
e47c68e9 4210 i915_gem_object_flush_cpu_write_domain(obj);
673a394b
EA		 4211 args->offset = obj_priv->gtt_offset;
4212 drm_gem_object_unreference(obj);
4213 mutex_unlock(&dev->struct_mutex);
4214
4215 return 0;
4216}
4217
4218int
4219i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4220 struct drm_file *file_priv)
4221{
4222 struct drm_i915_gem_pin *args = data;
4223 struct drm_gem_object *obj;
79e53945 4224 struct drm_i915_gem_object *obj_priv;
76c1dec1 4225 int ret;
673a394b
EA		 4226
4227 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4228 if (obj == NULL) {
4229 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4230 args->handle);
bf79cb91 4231 return -ENOENT;
673a394b
EA		 4232 }
4233
23010e43 4234 obj_priv = to_intel_bo(obj);
76c1dec1
CW		 4235
4236 ret = i915_mutex_lock_interruptible(dev);
4237 if (ret) {
4238 drm_gem_object_unreference_unlocked(obj);
4239 return ret;
4240 }
4241
79e53945
JB		 4242 if (obj_priv->pin_filp != file_priv) {
4243 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4244 args->handle);
4245 drm_gem_object_unreference(obj);
4246 mutex_unlock(&dev->struct_mutex);
4247 return -EINVAL;
4248 }
4249 obj_priv->user_pin_count--;
4250 if (obj_priv->user_pin_count == 0) {
4251 obj_priv->pin_filp = NULL;
4252 i915_gem_object_unpin(obj);
4253 }
673a394b
EA		 4254
4255 drm_gem_object_unreference(obj);
4256 mutex_unlock(&dev->struct_mutex);
4257 return 0;
4258}
4259
4260int
4261i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4262 struct drm_file *file_priv)
4263{
4264 struct drm_i915_gem_busy *args = data;
4265 struct drm_gem_object *obj;
4266 struct drm_i915_gem_object *obj_priv;
30dbf0c0
CW		 4267 int ret;
4268
673a394b
EA		 4269 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4270 if (obj == NULL) {
4271 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4272 args->handle);
bf79cb91 4273 return -ENOENT;
673a394b
EA		 4274 }
4275
76c1dec1
CW		 4276 ret = i915_mutex_lock_interruptible(dev);
4277 if (ret) {
4278 drm_gem_object_unreference_unlocked(obj);
4279 return ret;
30dbf0c0
CW		 4280 }
4281
0be555b6
CW		 4282 /* Count all active objects as busy, even if they are currently not used
4283 * by the gpu. Users of this interface expect objects to eventually
4284 * become non-busy without any further actions, therefore emit any
4285 * necessary flushes here.
c4de0a5d 4286 */
0be555b6
CW		 4287 obj_priv = to_intel_bo(obj);
4288 args->busy = obj_priv->active;
4289 if (args->busy) {
4290 /* Unconditionally flush objects, even when the gpu still uses this
4291 * object. Userspace calling this function indicates that it wants to
4292 * use this buffer rather sooner than later, so issuing the required
4293 * flush earlier is beneficial.
4294 */
c78ec30b
CW		 4295 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4296 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 4297 obj_priv->ring,
4298 0, obj->write_domain);
0be555b6
CW		 4299
4300 /* Update the active list for the hardware's current position.
4301 * Otherwise this only updates on a delayed timer or when irqs
4302 * are actually unmasked, and our working set ends up being
4303 * larger than required.
4304 */
4305 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4306
4307 args->busy = obj_priv->active;
4308 }
673a394b
EA		 4309
4310 drm_gem_object_unreference(obj);
4311 mutex_unlock(&dev->struct_mutex);
76c1dec1 4312 return 0;
673a394b
EA		 4313}
4314
4315int
4316i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4317 struct drm_file *file_priv)
4318{
4319 return i915_gem_ring_throttle(dev, file_priv);
4320}
4321
3ef94daa
CW		 4322int
4323i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4324 struct drm_file *file_priv)
4325{
4326 struct drm_i915_gem_madvise *args = data;
4327 struct drm_gem_object *obj;
4328 struct drm_i915_gem_object *obj_priv;
76c1dec1 4329 int ret;
3ef94daa
CW		 4330
4331 switch (args->madv) {
4332 case I915_MADV_DONTNEED:
4333 case I915_MADV_WILLNEED:
4334 break;
4335 default:
4336 return -EINVAL;
4337 }
4338
4339 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4340 if (obj == NULL) {
4341 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4342 args->handle);
bf79cb91 4343 return -ENOENT;
3ef94daa 4344 }
23010e43 4345 obj_priv = to_intel_bo(obj);
3ef94daa 4346
76c1dec1
CW		 4347 ret = i915_mutex_lock_interruptible(dev);
4348 if (ret) {
4349 drm_gem_object_unreference_unlocked(obj);
4350 return ret;
4351 }
4352
3ef94daa
CW		 4353 if (obj_priv->pin_count) {
4354 drm_gem_object_unreference(obj);
4355 mutex_unlock(&dev->struct_mutex);
4356
4357 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4358 return -EINVAL;
4359 }
4360
bb6baf76
CW		 4361 if (obj_priv->madv != __I915_MADV_PURGED)
4362 obj_priv->madv = args->madv;
3ef94daa 4363
2d7ef395
CW		 4364 /* if the object is no longer bound, discard its backing storage */
4365 if (i915_gem_object_is_purgeable(obj_priv) &&
4366 obj_priv->gtt_space == NULL)
4367 i915_gem_object_truncate(obj);
4368
bb6baf76
CW		 4369 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4370
3ef94daa
CW		 4371 drm_gem_object_unreference(obj);
4372 mutex_unlock(&dev->struct_mutex);
4373
4374 return 0;
4375}
4376
ac52bc56
DV		 4377struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4378 size_t size)
4379{
c397b908 4380 struct drm_i915_gem_object *obj;
ac52bc56 4381
c397b908
DV		 4382 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4383 if (obj == NULL)
4384 return NULL;
673a394b 4385
c397b908
DV		 4386 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4387 kfree(obj);
4388 return NULL;
4389 }
673a394b 4390
c397b908
DV		 4391 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4392 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
673a394b 4393
c397b908 4394 obj->agp_type = AGP_USER_MEMORY;
62b8b215 4395 obj->base.driver_private = NULL;
c397b908
DV		 4396 obj->fence_reg = I915_FENCE_REG_NONE;
4397 INIT_LIST_HEAD(&obj->list);
4398 INIT_LIST_HEAD(&obj->gpu_write_list);
c397b908 4399 obj->madv = I915_MADV_WILLNEED;
de151cf6 4400
c397b908
DV		 4401 trace_i915_gem_object_create(&obj->base);
4402
4403 return &obj->base;
4404}
4405
4406int i915_gem_init_object(struct drm_gem_object *obj)
4407{
4408 BUG();
de151cf6 4409
673a394b
EA		 4410 return 0;
4411}
4412
be72615b 4413static void i915_gem_free_object_tail(struct drm_gem_object *obj)
673a394b 4414{
de151cf6 4415 struct drm_device *dev = obj->dev;
be72615b 4416 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 4417 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
be72615b 4418 int ret;
673a394b 4419
be72615b
CW		 4420 ret = i915_gem_object_unbind(obj);
4421 if (ret == -ERESTARTSYS) {
4422 list_move(&obj_priv->list,
4423 &dev_priv->mm.deferred_free_list);
4424 return;
4425 }
673a394b 4426
7e616158
CW		 4427 if (obj_priv->mmap_offset)
4428 i915_gem_free_mmap_offset(obj);
de151cf6 4429
c397b908
DV		 4430 drm_gem_object_release(obj);
4431
9a298b2a 4432 kfree(obj_priv->page_cpu_valid);
280b713b 4433 kfree(obj_priv->bit_17);
c397b908 4434 kfree(obj_priv);
673a394b
EA		 4435}
4436
be72615b
CW		 4437void i915_gem_free_object(struct drm_gem_object *obj)
4438{
4439 struct drm_device *dev = obj->dev;
4440 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4441
4442 trace_i915_gem_object_destroy(obj);
4443
4444 while (obj_priv->pin_count > 0)
4445 i915_gem_object_unpin(obj);
4446
4447 if (obj_priv->phys_obj)
4448 i915_gem_detach_phys_object(dev, obj);
4449
4450 i915_gem_free_object_tail(obj);
4451}
4452
29105ccc
CW		 4453int
4454i915_gem_idle(struct drm_device *dev)
4455{
4456 drm_i915_private_t *dev_priv = dev->dev_private;
4457 int ret;
28dfe52a 4458
29105ccc 4459 mutex_lock(&dev->struct_mutex);
1c5d22f7 4460
8187a2b7 4461 if (dev_priv->mm.suspended ||
d1b851fc
ZN		 4462 (dev_priv->render_ring.gem_object == NULL) ||
4463 (HAS_BSD(dev) &&
4464 dev_priv->bsd_ring.gem_object == NULL)) {
29105ccc
CW		 4465 mutex_unlock(&dev->struct_mutex);
4466 return 0;
28dfe52a
EA		 4467 }
4468
29105ccc 4469 ret = i915_gpu_idle(dev);
6dbe2772
KP		 4470 if (ret) {
4471 mutex_unlock(&dev->struct_mutex);
673a394b 4472 return ret;
6dbe2772 4473 }
673a394b 4474
29105ccc
CW		 4475 /* Under UMS, be paranoid and evict. */
4476 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
b47eb4a2 4477 ret = i915_gem_evict_inactive(dev);
29105ccc
CW		 4478 if (ret) {
4479 mutex_unlock(&dev->struct_mutex);
4480 return ret;
4481 }
4482 }
4483
4484 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4485 * We need to replace this with a semaphore, or something.
4486 * And not confound mm.suspended!
4487 */
4488 dev_priv->mm.suspended = 1;
bc0c7f14 4489 del_timer_sync(&dev_priv->hangcheck_timer);
29105ccc
CW		 4490
4491 i915_kernel_lost_context(dev);
6dbe2772 4492 i915_gem_cleanup_ringbuffer(dev);
29105ccc 4493
6dbe2772
KP		 4494 mutex_unlock(&dev->struct_mutex);
4495
29105ccc
CW		 4496 /* Cancel the retire work handler, which should be idle now. */
4497 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4498
673a394b
EA		 4499 return 0;
4500}
4501
e552eb70
JB		 4502/*
4503 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4504 * over cache flushing.
4505 */
8187a2b7 4506static int
e552eb70
JB		 4507i915_gem_init_pipe_control(struct drm_device *dev)
4508{
4509 drm_i915_private_t *dev_priv = dev->dev_private;
4510 struct drm_gem_object *obj;
4511 struct drm_i915_gem_object *obj_priv;
4512 int ret;
4513
34dc4d44 4514 obj = i915_gem_alloc_object(dev, 4096);
e552eb70
JB		 4515 if (obj == NULL) {
4516 DRM_ERROR("Failed to allocate seqno page\n");
4517 ret = -ENOMEM;
4518 goto err;
4519 }
4520 obj_priv = to_intel_bo(obj);
4521 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4522
4523 ret = i915_gem_object_pin(obj, 4096);
4524 if (ret)
4525 goto err_unref;
4526
4527 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4528 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4529 if (dev_priv->seqno_page == NULL)
4530 goto err_unpin;
4531
4532 dev_priv->seqno_obj = obj;
4533 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4534
4535 return 0;
4536
4537err_unpin:
4538 i915_gem_object_unpin(obj);
4539err_unref:
4540 drm_gem_object_unreference(obj);
4541err:
4542 return ret;
4543}
4544
8187a2b7
ZN		 4545
4546static void
e552eb70
JB		 4547i915_gem_cleanup_pipe_control(struct drm_device *dev)
4548{
4549 drm_i915_private_t *dev_priv = dev->dev_private;
4550 struct drm_gem_object *obj;
4551 struct drm_i915_gem_object *obj_priv;
4552
4553 obj = dev_priv->seqno_obj;
4554 obj_priv = to_intel_bo(obj);
4555 kunmap(obj_priv->pages[0]);
4556 i915_gem_object_unpin(obj);
4557 drm_gem_object_unreference(obj);
4558 dev_priv->seqno_obj = NULL;
4559
4560 dev_priv->seqno_page = NULL;
673a394b
EA		 4561}
4562
8187a2b7
ZN		 4563int
4564i915_gem_init_ringbuffer(struct drm_device *dev)
4565{
4566 drm_i915_private_t *dev_priv = dev->dev_private;
4567 int ret;
68f95ba9 4568
8187a2b7
ZN		 4569 if (HAS_PIPE_CONTROL(dev)) {
4570 ret = i915_gem_init_pipe_control(dev);
4571 if (ret)
4572 return ret;
4573 }
68f95ba9 4574
5c1143bb 4575 ret = intel_init_render_ring_buffer(dev);
68f95ba9
CW		 4576 if (ret)
4577 goto cleanup_pipe_control;
4578
4579 if (HAS_BSD(dev)) {
5c1143bb 4580 ret = intel_init_bsd_ring_buffer(dev);
68f95ba9
CW		 4581 if (ret)
4582 goto cleanup_render_ring;
d1b851fc 4583 }
68f95ba9 4584
6f392d54
CW		 4585 dev_priv->next_seqno = 1;
4586
68f95ba9
CW		 4587 return 0;
4588
4589cleanup_render_ring:
4590 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4591cleanup_pipe_control:
4592 if (HAS_PIPE_CONTROL(dev))
4593 i915_gem_cleanup_pipe_control(dev);
8187a2b7
ZN		 4594 return ret;
4595}
4596
4597void
4598i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4599{
4600 drm_i915_private_t *dev_priv = dev->dev_private;
4601
4602 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
d1b851fc
ZN		 4603 if (HAS_BSD(dev))
4604 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
8187a2b7
ZN		 4605 if (HAS_PIPE_CONTROL(dev))
4606 i915_gem_cleanup_pipe_control(dev);
4607}
4608
673a394b
EA		 4609int
4610i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4611 struct drm_file *file_priv)
4612{
4613 drm_i915_private_t *dev_priv = dev->dev_private;
4614 int ret;
4615
79e53945
JB		 4616 if (drm_core_check_feature(dev, DRIVER_MODESET))
4617 return 0;
4618
ba1234d1 4619 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 4620 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 4621 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 4622 }
4623
673a394b 4624 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 4625 dev_priv->mm.suspended = 0;
4626
4627 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 4628 if (ret != 0) {
4629 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 4630 return ret;
d816f6ac 4631 }
9bb2d6f9 4632
852835f3 4633 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
d1b851fc 4634 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
673a394b
EA		 4635 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4636 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
852835f3 4637 BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
d1b851fc 4638 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
673a394b 4639 mutex_unlock(&dev->struct_mutex);
dbb19d30 4640
5f35308b
CW		 4641 ret = drm_irq_install(dev);
4642 if (ret)
4643 goto cleanup_ringbuffer;
dbb19d30 4644
673a394b 4645 return 0;
5f35308b
CW		 4646
4647cleanup_ringbuffer:
4648 mutex_lock(&dev->struct_mutex);
4649 i915_gem_cleanup_ringbuffer(dev);
4650 dev_priv->mm.suspended = 1;
4651 mutex_unlock(&dev->struct_mutex);
4652
4653 return ret;
673a394b
EA		 4654}
4655
4656int
4657i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4658 struct drm_file *file_priv)
4659{
79e53945
JB		 4660 if (drm_core_check_feature(dev, DRIVER_MODESET))
4661 return 0;
4662
dbb19d30 4663 drm_irq_uninstall(dev);
e6890f6f 4664 return i915_gem_idle(dev);
673a394b
EA		 4665}
4666
4667void
4668i915_gem_lastclose(struct drm_device *dev)
4669{
4670 int ret;
673a394b 4671
e806b495
EA		 4672 if (drm_core_check_feature(dev, DRIVER_MODESET))
4673 return;
4674
6dbe2772
KP		 4675 ret = i915_gem_idle(dev);
4676 if (ret)
4677 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 4678}
4679
4680void
4681i915_gem_load(struct drm_device *dev)
4682{
b5aa8a0f 4683 int i;
673a394b
EA		 4684 drm_i915_private_t *dev_priv = dev->dev_private;
4685
673a394b 4686 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
99fcb766 4687 INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
673a394b 4688 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
f13d3f73 4689 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
a09ba7fa 4690 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
be72615b 4691 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
852835f3
ZN		 4692 INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4693 INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
d1b851fc
ZN		 4694 if (HAS_BSD(dev)) {
4695 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4696 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4697 }
007cc8ac
DV		 4698 for (i = 0; i < 16; i++)
4699 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
673a394b
EA		 4700 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4701 i915_gem_retire_work_handler);
30dbf0c0 4702 init_completion(&dev_priv->error_completion);
31169714
CW		 4703 spin_lock(&shrink_list_lock);
4704 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4705 spin_unlock(&shrink_list_lock);
4706
94400120
DA		 4707 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4708 if (IS_GEN3(dev)) {
4709 u32 tmp = I915_READ(MI_ARB_STATE);
4710 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4711 /* arb state is a masked write, so set bit + bit in mask */
4712 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4713 I915_WRITE(MI_ARB_STATE, tmp);
4714 }
4715 }
4716
de151cf6 4717 /* Old X drivers will take 0-2 for front, back, depth buffers */
b397c836
EA		 4718 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4719 dev_priv->fence_reg_start = 3;
de151cf6 4720
a6c45cf0 4721 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 4722 dev_priv->num_fence_regs = 16;
4723 else
4724 dev_priv->num_fence_regs = 8;
4725
b5aa8a0f 4726 /* Initialize fence registers to zero */
a6c45cf0
CW		 4727 switch (INTEL_INFO(dev)->gen) {
4728 case 6:
4729 for (i = 0; i < 16; i++)
4730 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4731 break;
4732 case 5:
4733 case 4:
b5aa8a0f
GH		 4734 for (i = 0; i < 16; i++)
4735 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
a6c45cf0
CW		 4736 break;
4737 case 3:
b5aa8a0f
GH		 4738 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4739 for (i = 0; i < 8; i++)
4740 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
a6c45cf0
CW		 4741 case 2:
4742 for (i = 0; i < 8; i++)
4743 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4744 break;
b5aa8a0f 4745 }
673a394b 4746 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 4747 init_waitqueue_head(&dev_priv->pending_flip_queue);
673a394b 4748}
71acb5eb
DA		 4749
4750/*
4751 * Create a physically contiguous memory object for this object
4752 * e.g. for cursor + overlay regs
4753 */
995b6762
CW		 4754static int i915_gem_init_phys_object(struct drm_device *dev,
4755 int id, int size, int align)
71acb5eb
DA		 4756{
4757 drm_i915_private_t *dev_priv = dev->dev_private;
4758 struct drm_i915_gem_phys_object *phys_obj;
4759 int ret;
4760
4761 if (dev_priv->mm.phys_objs[id - 1] || !size)
4762 return 0;
4763
9a298b2a 4764 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 4765 if (!phys_obj)
4766 return -ENOMEM;
4767
4768 phys_obj->id = id;
4769
6eeefaf3 4770 phys_obj->handle = drm_pci_alloc(dev, size, align);
71acb5eb
DA		 4771 if (!phys_obj->handle) {
4772 ret = -ENOMEM;
4773 goto kfree_obj;
4774 }
4775#ifdef CONFIG_X86
4776 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4777#endif
4778
4779 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4780
4781 return 0;
4782kfree_obj:
9a298b2a 4783 kfree(phys_obj);
71acb5eb
DA		 4784 return ret;
4785}
4786
995b6762 4787static void i915_gem_free_phys_object(struct drm_device *dev, int id)
71acb5eb
DA		 4788{
4789 drm_i915_private_t *dev_priv = dev->dev_private;
4790 struct drm_i915_gem_phys_object *phys_obj;
4791
4792 if (!dev_priv->mm.phys_objs[id - 1])
4793 return;
4794
4795 phys_obj = dev_priv->mm.phys_objs[id - 1];
4796 if (phys_obj->cur_obj) {
4797 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4798 }
4799
4800#ifdef CONFIG_X86
4801 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4802#endif
4803 drm_pci_free(dev, phys_obj->handle);
4804 kfree(phys_obj);
4805 dev_priv->mm.phys_objs[id - 1] = NULL;
4806}
4807
4808void i915_gem_free_all_phys_object(struct drm_device *dev)
4809{
4810 int i;
4811
260883c8 4812 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 4813 i915_gem_free_phys_object(dev, i);
4814}
4815
4816void i915_gem_detach_phys_object(struct drm_device *dev,
4817 struct drm_gem_object *obj)
4818{
4819 struct drm_i915_gem_object *obj_priv;
4820 int i;
4821 int ret;
4822 int page_count;
4823
23010e43 4824 obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4825 if (!obj_priv->phys_obj)
4826 return;
4827
4bdadb97 4828 ret = i915_gem_object_get_pages(obj, 0);
71acb5eb
DA		 4829 if (ret)
4830 goto out;
4831
4832 page_count = obj->size / PAGE_SIZE;
4833
4834 for (i = 0; i < page_count; i++) {
856fa198 4835 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4836 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4837
4838 memcpy(dst, src, PAGE_SIZE);
4839 kunmap_atomic(dst, KM_USER0);
4840 }
856fa198 4841 drm_clflush_pages(obj_priv->pages, page_count);
71acb5eb 4842 drm_agp_chipset_flush(dev);
d78b47b9
CW		 4843
4844 i915_gem_object_put_pages(obj);
71acb5eb
DA		 4845out:
4846 obj_priv->phys_obj->cur_obj = NULL;
4847 obj_priv->phys_obj = NULL;
4848}
4849
4850int
4851i915_gem_attach_phys_object(struct drm_device *dev,
6eeefaf3
CW		 4852 struct drm_gem_object *obj,
4853 int id,
4854 int align)
71acb5eb
DA		 4855{
4856 drm_i915_private_t *dev_priv = dev->dev_private;
4857 struct drm_i915_gem_object *obj_priv;
4858 int ret = 0;
4859 int page_count;
4860 int i;
4861
4862 if (id > I915_MAX_PHYS_OBJECT)
4863 return -EINVAL;
4864
23010e43 4865 obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4866
4867 if (obj_priv->phys_obj) {
4868 if (obj_priv->phys_obj->id == id)
4869 return 0;
4870 i915_gem_detach_phys_object(dev, obj);
4871 }
4872
71acb5eb
DA		 4873 /* create a new object */
4874 if (!dev_priv->mm.phys_objs[id - 1]) {
4875 ret = i915_gem_init_phys_object(dev, id,
6eeefaf3 4876 obj->size, align);
71acb5eb 4877 if (ret) {
aeb565df 4878 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
71acb5eb
DA		 4879 goto out;
4880 }
4881 }
4882
4883 /* bind to the object */
4884 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4885 obj_priv->phys_obj->cur_obj = obj;
4886
4bdadb97 4887 ret = i915_gem_object_get_pages(obj, 0);
71acb5eb
DA		 4888 if (ret) {
4889 DRM_ERROR("failed to get page list\n");
4890 goto out;
4891 }
4892
4893 page_count = obj->size / PAGE_SIZE;
4894
4895 for (i = 0; i < page_count; i++) {
856fa198 4896 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4897 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4898
4899 memcpy(dst, src, PAGE_SIZE);
4900 kunmap_atomic(src, KM_USER0);
4901 }
4902
d78b47b9
CW		 4903 i915_gem_object_put_pages(obj);
4904
71acb5eb
DA		 4905 return 0;
4906out:
4907 return ret;
4908}
4909
4910static int
4911i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4912 struct drm_i915_gem_pwrite *args,
4913 struct drm_file *file_priv)
4914{
23010e43 4915 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4916 void *obj_addr;
4917 int ret;
4918 char __user *user_data;
4919
4920 user_data = (char __user *) (uintptr_t) args->data_ptr;
4921 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4922
44d98a61 4923 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
71acb5eb
DA		 4924 ret = copy_from_user(obj_addr, user_data, args->size);
4925 if (ret)
4926 return -EFAULT;
4927
4928 drm_agp_chipset_flush(dev);
4929 return 0;
4930}
b962442e 4931
f787a5f5 4932void i915_gem_release(struct drm_device *dev, struct drm_file *file)
b962442e 4933{
f787a5f5 4934 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e
EA		 4935
4936 /* Clean up our request list when the client is going away, so that
4937 * later retire_requests won't dereference our soon-to-be-gone
4938 * file_priv.
4939 */
1c25595f 4940 spin_lock(&file_priv->mm.lock);
f787a5f5
CW		 4941 while (!list_empty(&file_priv->mm.request_list)) {
4942 struct drm_i915_gem_request *request;
4943
4944 request = list_first_entry(&file_priv->mm.request_list,
4945 struct drm_i915_gem_request,
4946 client_list);
4947 list_del(&request->client_list);
4948 request->file_priv = NULL;
4949 }
1c25595f 4950 spin_unlock(&file_priv->mm.lock);
b962442e 4951}
31169714 4952
1637ef41
CW		 4953static int
4954i915_gpu_is_active(struct drm_device *dev)
4955{
4956 drm_i915_private_t *dev_priv = dev->dev_private;
4957 int lists_empty;
4958
1637ef41 4959 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
852835f3 4960 list_empty(&dev_priv->render_ring.active_list);
d1b851fc
ZN		 4961 if (HAS_BSD(dev))
4962 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
1637ef41
CW		 4963
4964 return !lists_empty;
4965}
4966
31169714 4967static int
7f8275d0 4968i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
31169714
CW		 4969{
4970 drm_i915_private_t *dev_priv, *next_dev;
4971 struct drm_i915_gem_object *obj_priv, *next_obj;
4972 int cnt = 0;
4973 int would_deadlock = 1;
4974
4975 /* "fast-path" to count number of available objects */
4976 if (nr_to_scan == 0) {
4977 spin_lock(&shrink_list_lock);
4978 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4979 struct drm_device *dev = dev_priv->dev;
4980
4981 if (mutex_trylock(&dev->struct_mutex)) {
4982 list_for_each_entry(obj_priv,
4983 &dev_priv->mm.inactive_list,
4984 list)
4985 cnt++;
4986 mutex_unlock(&dev->struct_mutex);
4987 }
4988 }
4989 spin_unlock(&shrink_list_lock);
4990
4991 return (cnt / 100) * sysctl_vfs_cache_pressure;
4992 }
4993
4994 spin_lock(&shrink_list_lock);
4995
1637ef41 4996rescan:
31169714
CW		 4997 /* first scan for clean buffers */
4998 list_for_each_entry_safe(dev_priv, next_dev,
4999 &shrink_list, mm.shrink_list) {
5000 struct drm_device *dev = dev_priv->dev;
5001
5002 if (! mutex_trylock(&dev->struct_mutex))
5003 continue;
5004
5005 spin_unlock(&shrink_list_lock);
b09a1fec 5006 i915_gem_retire_requests(dev);
31169714
CW		 5007
5008 list_for_each_entry_safe(obj_priv, next_obj,
5009 &dev_priv->mm.inactive_list,
5010 list) {
5011 if (i915_gem_object_is_purgeable(obj_priv)) {
a8089e84 5012 i915_gem_object_unbind(&obj_priv->base);
31169714
CW		 5013 if (--nr_to_scan <= 0)
5014 break;
5015 }
5016 }
5017
5018 spin_lock(&shrink_list_lock);
5019 mutex_unlock(&dev->struct_mutex);
5020
963b4836
CW		 5021 would_deadlock = 0;
5022
31169714
CW		 5023 if (nr_to_scan <= 0)
5024 break;
5025 }
5026
5027 /* second pass, evict/count anything still on the inactive list */
5028 list_for_each_entry_safe(dev_priv, next_dev,
5029 &shrink_list, mm.shrink_list) {
5030 struct drm_device *dev = dev_priv->dev;
5031
5032 if (! mutex_trylock(&dev->struct_mutex))
5033 continue;
5034
5035 spin_unlock(&shrink_list_lock);
5036
5037 list_for_each_entry_safe(obj_priv, next_obj,
5038 &dev_priv->mm.inactive_list,
5039 list) {
5040 if (nr_to_scan > 0) {
a8089e84 5041 i915_gem_object_unbind(&obj_priv->base);
31169714
CW		 5042 nr_to_scan--;
5043 } else
5044 cnt++;
5045 }
5046
5047 spin_lock(&shrink_list_lock);
5048 mutex_unlock(&dev->struct_mutex);
5049
5050 would_deadlock = 0;
5051 }
5052
1637ef41
CW		 5053 if (nr_to_scan) {
5054 int active = 0;
5055
5056 /*
5057 * We are desperate for pages, so as a last resort, wait
5058 * for the GPU to finish and discard whatever we can.
5059 * This has a dramatic impact to reduce the number of
5060 * OOM-killer events whilst running the GPU aggressively.
5061 */
5062 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5063 struct drm_device *dev = dev_priv->dev;
5064
5065 if (!mutex_trylock(&dev->struct_mutex))
5066 continue;
5067
5068 spin_unlock(&shrink_list_lock);
5069
5070 if (i915_gpu_is_active(dev)) {
5071 i915_gpu_idle(dev);
5072 active++;
5073 }
5074
5075 spin_lock(&shrink_list_lock);
5076 mutex_unlock(&dev->struct_mutex);
5077 }
5078
5079 if (active)
5080 goto rescan;
5081 }
5082
31169714
CW		 5083 spin_unlock(&shrink_list_lock);
5084
5085 if (would_deadlock)
5086 return -1;
5087 else if (cnt > 0)
5088 return (cnt / 100) * sysctl_vfs_cache_pressure;
5089 else
5090 return 0;
5091}
5092
5093static struct shrinker shrinker = {
5094 .shrink = i915_gem_shrink,
5095 .seeks = DEFAULT_SEEKS,
5096};
5097
5098__init void
5099i915_gem_shrinker_init(void)
5100{
5101 register_shrinker(&shrinker);
5102}
5103
5104__exit void
5105i915_gem_shrinker_exit(void)
5106{
5107 unregister_shrinker(&shrinker);
5108}
Linux kernel - Deliverables
This page took 0.506159 seconds and 5 git commands to generate.