projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drivers: gpu: drm: i915: Fix a typo.
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
5a0e3ad6 34#include <linux/slab.h>
673a394b 35#include <linux/swap.h>
79e53945 36#include <linux/pci.h>
f8f235e5 37#include <linux/intel-gtt.h>
673a394b 38
0108a3ed 39static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
ba3d8d74
DV		 40
41static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
42 bool pipelined);
e47c68e9
EA		 43static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
44static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
e47c68e9
EA		 45static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
46 int write);
47static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
48 uint64_t offset,
49 uint64_t size);
50static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
2cf34d7b
CW		 51static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
52 bool interruptible);
de151cf6
JB		 53static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
54 unsigned alignment);
de151cf6 55static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
71acb5eb
DA		 56static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
57 struct drm_i915_gem_pwrite *args,
58 struct drm_file *file_priv);
be72615b 59static void i915_gem_free_object_tail(struct drm_gem_object *obj);
673a394b 60
5cdf5881
CW		 61static int
62i915_gem_object_get_pages(struct drm_gem_object *obj,
63 gfp_t gfpmask);
64
65static void
66i915_gem_object_put_pages(struct drm_gem_object *obj);
67
31169714
CW		 68static LIST_HEAD(shrink_list);
69static DEFINE_SPINLOCK(shrink_list_lock);
70
73aa808f
CW		 71/* some bookkeeping */
72static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
73 size_t size)
74{
75 dev_priv->mm.object_count++;
76 dev_priv->mm.object_memory += size;
77}
78
79static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
80 size_t size)
81{
82 dev_priv->mm.object_count--;
83 dev_priv->mm.object_memory -= size;
84}
85
86static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
87 size_t size)
88{
89 dev_priv->mm.gtt_count++;
90 dev_priv->mm.gtt_memory += size;
91}
92
93static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
94 size_t size)
95{
96 dev_priv->mm.gtt_count--;
97 dev_priv->mm.gtt_memory -= size;
98}
99
100static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
101 size_t size)
102{
103 dev_priv->mm.pin_count++;
104 dev_priv->mm.pin_memory += size;
105}
106
107static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
108 size_t size)
109{
110 dev_priv->mm.pin_count--;
111 dev_priv->mm.pin_memory -= size;
112}
113
30dbf0c0
CW		 114int
115i915_gem_check_is_wedged(struct drm_device *dev)
116{
117 struct drm_i915_private *dev_priv = dev->dev_private;
118 struct completion *x = &dev_priv->error_completion;
119 unsigned long flags;
120 int ret;
121
122 if (!atomic_read(&dev_priv->mm.wedged))
123 return 0;
124
125 ret = wait_for_completion_interruptible(x);
126 if (ret)
127 return ret;
128
129 /* Success, we reset the GPU! */
130 if (!atomic_read(&dev_priv->mm.wedged))
131 return 0;
132
133 /* GPU is hung, bump the completion count to account for
134 * the token we just consumed so that we never hit zero and
135 * end up waiting upon a subsequent completion event that
136 * will never happen.
137 */
138 spin_lock_irqsave(&x->wait.lock, flags);
139 x->done++;
140 spin_unlock_irqrestore(&x->wait.lock, flags);
141 return -EIO;
142}
143
76c1dec1
CW		 144static int i915_mutex_lock_interruptible(struct drm_device *dev)
145{
146 struct drm_i915_private *dev_priv = dev->dev_private;
147 int ret;
148
149 ret = i915_gem_check_is_wedged(dev);
150 if (ret)
151 return ret;
152
153 ret = mutex_lock_interruptible(&dev->struct_mutex);
154 if (ret)
155 return ret;
156
157 if (atomic_read(&dev_priv->mm.wedged)) {
158 mutex_unlock(&dev->struct_mutex);
159 return -EAGAIN;
160 }
161
23bc5982 162 WARN_ON(i915_verify_lists(dev));
76c1dec1
CW		 163 return 0;
164}
30dbf0c0 165
7d1c4804
CW		 166static inline bool
167i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
168{
169 return obj_priv->gtt_space &&
170 !obj_priv->active &&
171 obj_priv->pin_count == 0;
172}
173
73aa808f
CW		 174int i915_gem_do_init(struct drm_device *dev,
175 unsigned long start,
79e53945 176 unsigned long end)
673a394b
EA		 177{
178 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 179
79e53945
JB		 180 if (start >= end ||
181 (start & (PAGE_SIZE - 1)) != 0 ||
182 (end & (PAGE_SIZE - 1)) != 0) {
673a394b
EA		 183 return -EINVAL;
184 }
185
79e53945
JB		 186 drm_mm_init(&dev_priv->mm.gtt_space, start,
187 end - start);
673a394b 188
73aa808f 189 dev_priv->mm.gtt_total = end - start;
79e53945
JB		 190
191 return 0;
192}
673a394b 193
79e53945
JB		 194int
195i915_gem_init_ioctl(struct drm_device *dev, void *data,
196 struct drm_file *file_priv)
197{
198 struct drm_i915_gem_init *args = data;
199 int ret;
200
201 mutex_lock(&dev->struct_mutex);
202 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
673a394b
EA		 203 mutex_unlock(&dev->struct_mutex);
204
79e53945 205 return ret;
673a394b
EA		 206}
207
5a125c3c
EA		 208int
209i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
210 struct drm_file *file_priv)
211{
73aa808f 212 struct drm_i915_private *dev_priv = dev->dev_private;
5a125c3c 213 struct drm_i915_gem_get_aperture *args = data;
5a125c3c
EA		 214
215 if (!(dev->driver->driver_features & DRIVER_GEM))
216 return -ENODEV;
217
73aa808f
CW		 218 mutex_lock(&dev->struct_mutex);
219 args->aper_size = dev_priv->mm.gtt_total;
220 args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
221 mutex_unlock(&dev->struct_mutex);
5a125c3c
EA		 222
223 return 0;
224}
225
673a394b
EA		 226
227/**
228 * Creates a new mm object and returns a handle to it.
229 */
230int
231i915_gem_create_ioctl(struct drm_device *dev, void *data,
232 struct drm_file *file_priv)
233{
234 struct drm_i915_gem_create *args = data;
235 struct drm_gem_object *obj;
a1a2d1d3
PP		 236 int ret;
237 u32 handle;
673a394b
EA		 238
239 args->size = roundup(args->size, PAGE_SIZE);
240
241 /* Allocate the new object */
ac52bc56 242 obj = i915_gem_alloc_object(dev, args->size);
673a394b
EA		 243 if (obj == NULL)
244 return -ENOMEM;
245
246 ret = drm_gem_handle_create(file_priv, obj, &handle);
1dfd9754 247 if (ret) {
202f2fef
CW		 248 drm_gem_object_release(obj);
249 i915_gem_info_remove_obj(dev->dev_private, obj->size);
250 kfree(obj);
673a394b 251 return ret;
1dfd9754 252 }
673a394b 253
202f2fef
CW		 254 /* drop reference from allocate - handle holds it now */
255 drm_gem_object_unreference(obj);
256 trace_i915_gem_object_create(obj);
257
1dfd9754 258 args->handle = handle;
673a394b
EA		 259 return 0;
260}
261
eb01459f
EA		 262static inline int
263fast_shmem_read(struct page **pages,
264 loff_t page_base, int page_offset,
265 char __user *data,
266 int length)
267{
b5e4feb6 268 char *vaddr;
4f27b75d 269 int ret;
eb01459f
EA		 270
271 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
4f27b75d 272 ret = __copy_to_user_inatomic(data, vaddr + page_offset, length);
eb01459f
EA		 273 kunmap_atomic(vaddr, KM_USER0);
274
4f27b75d 275 return ret;
eb01459f
EA		 276}
277
280b713b
EA		 278static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
279{
280 drm_i915_private_t *dev_priv = obj->dev->dev_private;
23010e43 281 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
280b713b
EA		 282
283 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
284 obj_priv->tiling_mode != I915_TILING_NONE;
285}
286
99a03df5 287static inline void
40123c1f
EA		 288slow_shmem_copy(struct page *dst_page,
289 int dst_offset,
290 struct page *src_page,
291 int src_offset,
292 int length)
293{
294 char *dst_vaddr, *src_vaddr;
295
99a03df5
CW		 296 dst_vaddr = kmap(dst_page);
297 src_vaddr = kmap(src_page);
40123c1f
EA		 298
299 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
300
99a03df5
CW		 301 kunmap(src_page);
302 kunmap(dst_page);
40123c1f
EA		 303}
304
99a03df5 305static inline void
280b713b
EA		 306slow_shmem_bit17_copy(struct page *gpu_page,
307 int gpu_offset,
308 struct page *cpu_page,
309 int cpu_offset,
310 int length,
311 int is_read)
312{
313 char *gpu_vaddr, *cpu_vaddr;
314
315 /* Use the unswizzled path if this page isn't affected. */
316 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
317 if (is_read)
318 return slow_shmem_copy(cpu_page, cpu_offset,
319 gpu_page, gpu_offset, length);
320 else
321 return slow_shmem_copy(gpu_page, gpu_offset,
322 cpu_page, cpu_offset, length);
323 }
324
99a03df5
CW		 325 gpu_vaddr = kmap(gpu_page);
326 cpu_vaddr = kmap(cpu_page);
280b713b
EA		 327
328 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
329 * XORing with the other bits (A9 for Y, A9 and A10 for X)
330 */
331 while (length > 0) {
332 int cacheline_end = ALIGN(gpu_offset + 1, 64);
333 int this_length = min(cacheline_end - gpu_offset, length);
334 int swizzled_gpu_offset = gpu_offset ^ 64;
335
336 if (is_read) {
337 memcpy(cpu_vaddr + cpu_offset,
338 gpu_vaddr + swizzled_gpu_offset,
339 this_length);
340 } else {
341 memcpy(gpu_vaddr + swizzled_gpu_offset,
342 cpu_vaddr + cpu_offset,
343 this_length);
344 }
345 cpu_offset += this_length;
346 gpu_offset += this_length;
347 length -= this_length;
348 }
349
99a03df5
CW		 350 kunmap(cpu_page);
351 kunmap(gpu_page);
280b713b
EA		 352}
353
eb01459f
EA		 354/**
355 * This is the fast shmem pread path, which attempts to copy_from_user directly
356 * from the backing pages of the object to the user's address space. On a
357 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
358 */
359static int
360i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
361 struct drm_i915_gem_pread *args,
362 struct drm_file *file_priv)
363{
23010e43 364 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
eb01459f
EA		 365 ssize_t remain;
366 loff_t offset, page_base;
367 char __user *user_data;
368 int page_offset, page_length;
eb01459f
EA		 369
370 user_data = (char __user *) (uintptr_t) args->data_ptr;
371 remain = args->size;
372
23010e43 373 obj_priv = to_intel_bo(obj);
eb01459f
EA		 374 offset = args->offset;
375
376 while (remain > 0) {
377 /* Operation in this page
378 *
379 * page_base = page offset within aperture
380 * page_offset = offset within page
381 * page_length = bytes to copy for this page
382 */
383 page_base = (offset & ~(PAGE_SIZE-1));
384 page_offset = offset & (PAGE_SIZE-1);
385 page_length = remain;
386 if ((page_offset + remain) > PAGE_SIZE)
387 page_length = PAGE_SIZE - page_offset;
388
4f27b75d
CW		 389 if (fast_shmem_read(obj_priv->pages,
390 page_base, page_offset,
391 user_data, page_length))
392 return -EFAULT;
eb01459f
EA		 393
394 remain -= page_length;
395 user_data += page_length;
396 offset += page_length;
397 }
398
4f27b75d 399 return 0;
eb01459f
EA		 400}
401
07f73f69
CW		 402static int
403i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
404{
405 int ret;
406
4bdadb97 407 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
07f73f69
CW		 408
409 /* If we've insufficient memory to map in the pages, attempt
410 * to make some space by throwing out some old buffers.
411 */
412 if (ret == -ENOMEM) {
413 struct drm_device *dev = obj->dev;
07f73f69 414
0108a3ed
DV		 415 ret = i915_gem_evict_something(dev, obj->size,
416 i915_gem_get_gtt_alignment(obj));
07f73f69
CW		 417 if (ret)
418 return ret;
419
4bdadb97 420 ret = i915_gem_object_get_pages(obj, 0);
07f73f69
CW		 421 }
422
423 return ret;
424}
425
eb01459f
EA		 426/**
427 * This is the fallback shmem pread path, which allocates temporary storage
428 * in kernel space to copy_to_user into outside of the struct_mutex, so we
429 * can copy out of the object's backing pages while holding the struct mutex
430 * and not take page faults.
431 */
432static int
433i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
434 struct drm_i915_gem_pread *args,
435 struct drm_file *file_priv)
436{
23010e43 437 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
eb01459f
EA		 438 struct mm_struct *mm = current->mm;
439 struct page **user_pages;
440 ssize_t remain;
441 loff_t offset, pinned_pages, i;
442 loff_t first_data_page, last_data_page, num_pages;
443 int shmem_page_index, shmem_page_offset;
444 int data_page_index, data_page_offset;
445 int page_length;
446 int ret;
447 uint64_t data_ptr = args->data_ptr;
280b713b 448 int do_bit17_swizzling;
eb01459f
EA		 449
450 remain = args->size;
451
452 /* Pin the user pages containing the data. We can't fault while
453 * holding the struct mutex, yet we want to hold it while
454 * dereferencing the user data.
455 */
456 first_data_page = data_ptr / PAGE_SIZE;
457 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
458 num_pages = last_data_page - first_data_page + 1;
459
4f27b75d 460 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
eb01459f
EA		 461 if (user_pages == NULL)
462 return -ENOMEM;
463
4f27b75d 464 mutex_unlock(&dev->struct_mutex);
eb01459f
EA		 465 down_read(&mm->mmap_sem);
466 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 467 num_pages, 1, 0, user_pages, NULL);
eb01459f 468 up_read(&mm->mmap_sem);
4f27b75d 469 mutex_lock(&dev->struct_mutex);
eb01459f
EA		 470 if (pinned_pages < num_pages) {
471 ret = -EFAULT;
4f27b75d 472 goto out;
eb01459f
EA		 473 }
474
4f27b75d
CW		 475 ret = i915_gem_object_set_cpu_read_domain_range(obj,
476 args->offset,
477 args->size);
07f73f69 478 if (ret)
4f27b75d 479 goto out;
eb01459f 480
4f27b75d 481 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
eb01459f 482
23010e43 483 obj_priv = to_intel_bo(obj);
eb01459f
EA		 484 offset = args->offset;
485
486 while (remain > 0) {
487 /* Operation in this page
488 *
489 * shmem_page_index = page number within shmem file
490 * shmem_page_offset = offset within page in shmem file
491 * data_page_index = page number in get_user_pages return
492 * data_page_offset = offset with data_page_index page.
493 * page_length = bytes to copy for this page
494 */
495 shmem_page_index = offset / PAGE_SIZE;
496 shmem_page_offset = offset & ~PAGE_MASK;
497 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
498 data_page_offset = data_ptr & ~PAGE_MASK;
499
500 page_length = remain;
501 if ((shmem_page_offset + page_length) > PAGE_SIZE)
502 page_length = PAGE_SIZE - shmem_page_offset;
503 if ((data_page_offset + page_length) > PAGE_SIZE)
504 page_length = PAGE_SIZE - data_page_offset;
505
280b713b 506 if (do_bit17_swizzling) {
99a03df5 507 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
280b713b 508 shmem_page_offset,
99a03df5
CW		 509 user_pages[data_page_index],
510 data_page_offset,
511 page_length,
512 1);
513 } else {
514 slow_shmem_copy(user_pages[data_page_index],
515 data_page_offset,
516 obj_priv->pages[shmem_page_index],
517 shmem_page_offset,
518 page_length);
280b713b 519 }
eb01459f
EA		 520
521 remain -= page_length;
522 data_ptr += page_length;
523 offset += page_length;
524 }
525
4f27b75d 526out:
eb01459f
EA		 527 for (i = 0; i < pinned_pages; i++) {
528 SetPageDirty(user_pages[i]);
529 page_cache_release(user_pages[i]);
530 }
8e7d2b2c 531 drm_free_large(user_pages);
eb01459f
EA		 532
533 return ret;
534}
535
673a394b
EA		 536/**
537 * Reads data from the object referenced by handle.
538 *
539 * On error, the contents of *data are undefined.
540 */
541int
542i915_gem_pread_ioctl(struct drm_device *dev, void *data,
543 struct drm_file *file_priv)
544{
545 struct drm_i915_gem_pread *args = data;
546 struct drm_gem_object *obj;
547 struct drm_i915_gem_object *obj_priv;
35b62a89 548 int ret = 0;
673a394b
EA		 549
550 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
551 if (obj == NULL)
bf79cb91 552 return -ENOENT;
23010e43 553 obj_priv = to_intel_bo(obj);
673a394b 554
4f27b75d
CW		 555 ret = i915_mutex_lock_interruptible(dev);
556 if (ret) {
557 drm_gem_object_unreference_unlocked(obj);
558 return ret;
559 }
560
7dcd2499
CW		 561 /* Bounds check source. */
562 if (args->offset > obj->size || args->size > obj->size - args->offset) {
ce9d419d 563 ret = -EINVAL;
35b62a89 564 goto out;
ce9d419d
CW		 565 }
566
35b62a89
CW		 567 if (args->size == 0)
568 goto out;
569
ce9d419d
CW		 570 if (!access_ok(VERIFY_WRITE,
571 (char __user *)(uintptr_t)args->data_ptr,
572 args->size)) {
573 ret = -EFAULT;
35b62a89 574 goto out;
673a394b
EA		 575 }
576
b5e4feb6
CW		 577 ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
578 args->size);
579 if (ret) {
580 ret = -EFAULT;
581 goto out;
582 }
583
4f27b75d
CW		 584 ret = i915_gem_object_get_pages_or_evict(obj);
585 if (ret)
586 goto out;
587
588 ret = i915_gem_object_set_cpu_read_domain_range(obj,
589 args->offset,
590 args->size);
591 if (ret)
592 goto out_put;
593
594 ret = -EFAULT;
595 if (!i915_gem_object_needs_bit17_swizzle(obj))
280b713b 596 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
4f27b75d
CW		 597 if (ret == -EFAULT)
598 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
673a394b 599
4f27b75d
CW		 600out_put:
601 i915_gem_object_put_pages(obj);
35b62a89 602out:
4f27b75d
CW		 603 drm_gem_object_unreference(obj);
604 mutex_unlock(&dev->struct_mutex);
eb01459f 605 return ret;
673a394b
EA		 606}
607
0839ccb8
KP		 608/* This is the fast write path which cannot handle
609 * page faults in the source data
9b7530cc 610 */
0839ccb8
KP		 611
612static inline int
613fast_user_write(struct io_mapping *mapping,
614 loff_t page_base, int page_offset,
615 char __user *user_data,
616 int length)
9b7530cc 617{
9b7530cc 618 char *vaddr_atomic;
0839ccb8 619 unsigned long unwritten;
9b7530cc 620
fca3ec01 621 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
0839ccb8
KP		 622 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
623 user_data, length);
fca3ec01 624 io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
fbd5a26d 625 return unwritten;
0839ccb8
KP		 626}
627
628/* Here's the write path which can sleep for
629 * page faults
630 */
631
ab34c226 632static inline void
3de09aa3
EA		 633slow_kernel_write(struct io_mapping *mapping,
634 loff_t gtt_base, int gtt_offset,
635 struct page *user_page, int user_offset,
636 int length)
0839ccb8 637{
ab34c226
CW		 638 char __iomem *dst_vaddr;
639 char *src_vaddr;
0839ccb8 640
ab34c226
CW		 641 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
642 src_vaddr = kmap(user_page);
643
644 memcpy_toio(dst_vaddr + gtt_offset,
645 src_vaddr + user_offset,
646 length);
647
648 kunmap(user_page);
649 io_mapping_unmap(dst_vaddr);
9b7530cc
LT		 650}
651
40123c1f
EA		 652static inline int
653fast_shmem_write(struct page **pages,
654 loff_t page_base, int page_offset,
655 char __user *data,
656 int length)
657{
b5e4feb6 658 char *vaddr;
fbd5a26d 659 int ret;
40123c1f
EA		 660
661 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
fbd5a26d 662 ret = __copy_from_user_inatomic(vaddr + page_offset, data, length);
40123c1f
EA		 663 kunmap_atomic(vaddr, KM_USER0);
664
fbd5a26d 665 return ret;
40123c1f
EA		 666}
667
3de09aa3
EA		 668/**
669 * This is the fast pwrite path, where we copy the data directly from the
670 * user into the GTT, uncached.
671 */
673a394b 672static int
3de09aa3
EA		 673i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
674 struct drm_i915_gem_pwrite *args,
675 struct drm_file *file_priv)
673a394b 676{
23010e43 677 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
0839ccb8 678 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 679 ssize_t remain;
0839ccb8 680 loff_t offset, page_base;
673a394b 681 char __user *user_data;
0839ccb8 682 int page_offset, page_length;
673a394b
EA		 683
684 user_data = (char __user *) (uintptr_t) args->data_ptr;
685 remain = args->size;
673a394b 686
23010e43 687 obj_priv = to_intel_bo(obj);
673a394b 688 offset = obj_priv->gtt_offset + args->offset;
673a394b
EA		 689
690 while (remain > 0) {
691 /* Operation in this page
692 *
0839ccb8
KP		 693 * page_base = page offset within aperture
694 * page_offset = offset within page
695 * page_length = bytes to copy for this page
673a394b 696 */
0839ccb8
KP		 697 page_base = (offset & ~(PAGE_SIZE-1));
698 page_offset = offset & (PAGE_SIZE-1);
699 page_length = remain;
700 if ((page_offset + remain) > PAGE_SIZE)
701 page_length = PAGE_SIZE - page_offset;
702
0839ccb8 703 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 704 * source page isn't available. Return the error and we'll
705 * retry in the slow path.
0839ccb8 706 */
fbd5a26d
CW		 707 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
708 page_offset, user_data, page_length))
709
710 return -EFAULT;
673a394b 711
0839ccb8
KP		 712 remain -= page_length;
713 user_data += page_length;
714 offset += page_length;
673a394b 715 }
673a394b 716
fbd5a26d 717 return 0;
673a394b
EA		 718}
719
3de09aa3
EA		 720/**
721 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
722 * the memory and maps it using kmap_atomic for copying.
723 *
724 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
725 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
726 */
3043c60c 727static int
3de09aa3
EA		 728i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
729 struct drm_i915_gem_pwrite *args,
730 struct drm_file *file_priv)
673a394b 731{
23010e43 732 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3de09aa3
EA		 733 drm_i915_private_t *dev_priv = dev->dev_private;
734 ssize_t remain;
735 loff_t gtt_page_base, offset;
736 loff_t first_data_page, last_data_page, num_pages;
737 loff_t pinned_pages, i;
738 struct page **user_pages;
739 struct mm_struct *mm = current->mm;
740 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 741 int ret;
3de09aa3
EA		 742 uint64_t data_ptr = args->data_ptr;
743
744 remain = args->size;
745
746 /* Pin the user pages containing the data. We can't fault while
747 * holding the struct mutex, and all of the pwrite implementations
748 * want to hold it while dereferencing the user data.
749 */
750 first_data_page = data_ptr / PAGE_SIZE;
751 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
752 num_pages = last_data_page - first_data_page + 1;
753
fbd5a26d 754 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
3de09aa3
EA		 755 if (user_pages == NULL)
756 return -ENOMEM;
757
fbd5a26d 758 mutex_unlock(&dev->struct_mutex);
3de09aa3
EA		 759 down_read(&mm->mmap_sem);
760 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
761 num_pages, 0, 0, user_pages, NULL);
762 up_read(&mm->mmap_sem);
fbd5a26d 763 mutex_lock(&dev->struct_mutex);
3de09aa3
EA		 764 if (pinned_pages < num_pages) {
765 ret = -EFAULT;
766 goto out_unpin_pages;
767 }
673a394b 768
3de09aa3
EA		 769 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
770 if (ret)
fbd5a26d 771 goto out_unpin_pages;
3de09aa3 772
23010e43 773 obj_priv = to_intel_bo(obj);
3de09aa3
EA		 774 offset = obj_priv->gtt_offset + args->offset;
775
776 while (remain > 0) {
777 /* Operation in this page
778 *
779 * gtt_page_base = page offset within aperture
780 * gtt_page_offset = offset within page in aperture
781 * data_page_index = page number in get_user_pages return
782 * data_page_offset = offset with data_page_index page.
783 * page_length = bytes to copy for this page
784 */
785 gtt_page_base = offset & PAGE_MASK;
786 gtt_page_offset = offset & ~PAGE_MASK;
787 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
788 data_page_offset = data_ptr & ~PAGE_MASK;
789
790 page_length = remain;
791 if ((gtt_page_offset + page_length) > PAGE_SIZE)
792 page_length = PAGE_SIZE - gtt_page_offset;
793 if ((data_page_offset + page_length) > PAGE_SIZE)
794 page_length = PAGE_SIZE - data_page_offset;
795
ab34c226
CW		 796 slow_kernel_write(dev_priv->mm.gtt_mapping,
797 gtt_page_base, gtt_page_offset,
798 user_pages[data_page_index],
799 data_page_offset,
800 page_length);
3de09aa3
EA		 801
802 remain -= page_length;
803 offset += page_length;
804 data_ptr += page_length;
805 }
806
3de09aa3
EA		 807out_unpin_pages:
808 for (i = 0; i < pinned_pages; i++)
809 page_cache_release(user_pages[i]);
8e7d2b2c 810 drm_free_large(user_pages);
3de09aa3
EA		 811
812 return ret;
813}
814
40123c1f
EA		 815/**
816 * This is the fast shmem pwrite path, which attempts to directly
817 * copy_from_user into the kmapped pages backing the object.
818 */
3043c60c 819static int
40123c1f
EA		 820i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
821 struct drm_i915_gem_pwrite *args,
822 struct drm_file *file_priv)
673a394b 823{
23010e43 824 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
40123c1f
EA		 825 ssize_t remain;
826 loff_t offset, page_base;
827 char __user *user_data;
828 int page_offset, page_length;
40123c1f
EA		 829
830 user_data = (char __user *) (uintptr_t) args->data_ptr;
831 remain = args->size;
673a394b 832
23010e43 833 obj_priv = to_intel_bo(obj);
40123c1f
EA		 834 offset = args->offset;
835 obj_priv->dirty = 1;
836
837 while (remain > 0) {
838 /* Operation in this page
839 *
840 * page_base = page offset within aperture
841 * page_offset = offset within page
842 * page_length = bytes to copy for this page
843 */
844 page_base = (offset & ~(PAGE_SIZE-1));
845 page_offset = offset & (PAGE_SIZE-1);
846 page_length = remain;
847 if ((page_offset + remain) > PAGE_SIZE)
848 page_length = PAGE_SIZE - page_offset;
849
fbd5a26d 850 if (fast_shmem_write(obj_priv->pages,
40123c1f 851 page_base, page_offset,
fbd5a26d
CW		 852 user_data, page_length))
853 return -EFAULT;
40123c1f
EA		 854
855 remain -= page_length;
856 user_data += page_length;
857 offset += page_length;
858 }
859
fbd5a26d 860 return 0;
40123c1f
EA		 861}
862
863/**
864 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
865 * the memory and maps it using kmap_atomic for copying.
866 *
867 * This avoids taking mmap_sem for faulting on the user's address while the
868 * struct_mutex is held.
869 */
870static int
871i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
872 struct drm_i915_gem_pwrite *args,
873 struct drm_file *file_priv)
874{
23010e43 875 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
40123c1f
EA		 876 struct mm_struct *mm = current->mm;
877 struct page **user_pages;
878 ssize_t remain;
879 loff_t offset, pinned_pages, i;
880 loff_t first_data_page, last_data_page, num_pages;
881 int shmem_page_index, shmem_page_offset;
882 int data_page_index, data_page_offset;
883 int page_length;
884 int ret;
885 uint64_t data_ptr = args->data_ptr;
280b713b 886 int do_bit17_swizzling;
40123c1f
EA		 887
888 remain = args->size;
889
890 /* Pin the user pages containing the data. We can't fault while
891 * holding the struct mutex, and all of the pwrite implementations
892 * want to hold it while dereferencing the user data.
893 */
894 first_data_page = data_ptr / PAGE_SIZE;
895 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
896 num_pages = last_data_page - first_data_page + 1;
897
4f27b75d 898 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
40123c1f
EA		 899 if (user_pages == NULL)
900 return -ENOMEM;
901
fbd5a26d 902 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 903 down_read(&mm->mmap_sem);
904 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
905 num_pages, 0, 0, user_pages, NULL);
906 up_read(&mm->mmap_sem);
fbd5a26d 907 mutex_lock(&dev->struct_mutex);
40123c1f
EA		 908 if (pinned_pages < num_pages) {
909 ret = -EFAULT;
fbd5a26d 910 goto out;
673a394b
EA		 911 }
912
fbd5a26d 913 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
07f73f69 914 if (ret)
fbd5a26d 915 goto out;
40123c1f 916
fbd5a26d 917 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
40123c1f 918
23010e43 919 obj_priv = to_intel_bo(obj);
673a394b 920 offset = args->offset;
40123c1f 921 obj_priv->dirty = 1;
673a394b 922
40123c1f
EA		 923 while (remain > 0) {
924 /* Operation in this page
925 *
926 * shmem_page_index = page number within shmem file
927 * shmem_page_offset = offset within page in shmem file
928 * data_page_index = page number in get_user_pages return
929 * data_page_offset = offset with data_page_index page.
930 * page_length = bytes to copy for this page
931 */
932 shmem_page_index = offset / PAGE_SIZE;
933 shmem_page_offset = offset & ~PAGE_MASK;
934 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
935 data_page_offset = data_ptr & ~PAGE_MASK;
936
937 page_length = remain;
938 if ((shmem_page_offset + page_length) > PAGE_SIZE)
939 page_length = PAGE_SIZE - shmem_page_offset;
940 if ((data_page_offset + page_length) > PAGE_SIZE)
941 page_length = PAGE_SIZE - data_page_offset;
942
280b713b 943 if (do_bit17_swizzling) {
99a03df5 944 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
280b713b
EA		 945 shmem_page_offset,
946 user_pages[data_page_index],
947 data_page_offset,
99a03df5
CW		 948 page_length,
949 0);
950 } else {
951 slow_shmem_copy(obj_priv->pages[shmem_page_index],
952 shmem_page_offset,
953 user_pages[data_page_index],
954 data_page_offset,
955 page_length);
280b713b 956 }
40123c1f
EA		 957
958 remain -= page_length;
959 data_ptr += page_length;
960 offset += page_length;
673a394b
EA		 961 }
962
fbd5a26d 963out:
40123c1f
EA		 964 for (i = 0; i < pinned_pages; i++)
965 page_cache_release(user_pages[i]);
8e7d2b2c 966 drm_free_large(user_pages);
673a394b 967
40123c1f 968 return ret;
673a394b
EA		 969}
970
971/**
972 * Writes data to the object referenced by handle.
973 *
974 * On error, the contents of the buffer that were to be modified are undefined.
975 */
976int
977i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
fbd5a26d 978 struct drm_file *file)
673a394b
EA		 979{
980 struct drm_i915_gem_pwrite *args = data;
981 struct drm_gem_object *obj;
982 struct drm_i915_gem_object *obj_priv;
983 int ret = 0;
984
fbd5a26d 985 obj = drm_gem_object_lookup(dev, file, args->handle);
673a394b 986 if (obj == NULL)
bf79cb91 987 return -ENOENT;
23010e43 988 obj_priv = to_intel_bo(obj);
673a394b 989
fbd5a26d
CW		 990 ret = i915_mutex_lock_interruptible(dev);
991 if (ret) {
992 drm_gem_object_unreference_unlocked(obj);
993 return ret;
994 }
995
7dcd2499
CW		 996 /* Bounds check destination. */
997 if (args->offset > obj->size || args->size > obj->size - args->offset) {
ce9d419d 998 ret = -EINVAL;
35b62a89 999 goto out;
ce9d419d
CW		 1000 }
1001
35b62a89
CW		 1002 if (args->size == 0)
1003 goto out;
1004
ce9d419d
CW		 1005 if (!access_ok(VERIFY_READ,
1006 (char __user *)(uintptr_t)args->data_ptr,
1007 args->size)) {
1008 ret = -EFAULT;
35b62a89 1009 goto out;
673a394b
EA		 1010 }
1011
b5e4feb6
CW		 1012 ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
1013 args->size);
1014 if (ret) {
1015 ret = -EFAULT;
1016 goto out;
1017 }
1018
673a394b
EA		 1019 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1020 * it would end up going through the fenced access, and we'll get
1021 * different detiling behavior between reading and writing.
1022 * pread/pwrite currently are reading and writing from the CPU
1023 * perspective, requiring manual detiling by the client.
1024 */
71acb5eb 1025 if (obj_priv->phys_obj)
fbd5a26d 1026 ret = i915_gem_phys_pwrite(dev, obj, args, file);
71acb5eb 1027 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
5cdf5881 1028 obj_priv->gtt_space &&
9b8c4a0b 1029 obj->write_domain != I915_GEM_DOMAIN_CPU) {
fbd5a26d
CW		 1030 ret = i915_gem_object_pin(obj, 0);
1031 if (ret)
1032 goto out;
1033
1034 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1035 if (ret)
1036 goto out_unpin;
1037
1038 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1039 if (ret == -EFAULT)
1040 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1041
1042out_unpin:
1043 i915_gem_object_unpin(obj);
40123c1f 1044 } else {
fbd5a26d
CW		 1045 ret = i915_gem_object_get_pages_or_evict(obj);
1046 if (ret)
1047 goto out;
673a394b 1048
fbd5a26d
CW		 1049 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1050 if (ret)
1051 goto out_put;
1052
1053 ret = -EFAULT;
1054 if (!i915_gem_object_needs_bit17_swizzle(obj))
1055 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1056 if (ret == -EFAULT)
1057 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1058
1059out_put:
1060 i915_gem_object_put_pages(obj);
1061 }
673a394b 1062
35b62a89 1063out:
fbd5a26d
CW		 1064 drm_gem_object_unreference(obj);
1065 mutex_unlock(&dev->struct_mutex);
673a394b
EA		 1066 return ret;
1067}
1068
1069/**
2ef7eeaa
EA		 1070 * Called when user space prepares to use an object with the CPU, either
1071 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 1072 */
1073int
1074i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1075 struct drm_file *file_priv)
1076{
a09ba7fa 1077 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 1078 struct drm_i915_gem_set_domain *args = data;
1079 struct drm_gem_object *obj;
652c393a 1080 struct drm_i915_gem_object *obj_priv;
2ef7eeaa
EA		 1081 uint32_t read_domains = args->read_domains;
1082 uint32_t write_domain = args->write_domain;
673a394b
EA		 1083 int ret;
1084
1085 if (!(dev->driver->driver_features & DRIVER_GEM))
1086 return -ENODEV;
1087
2ef7eeaa 1088 /* Only handle setting domains to types used by the CPU. */
21d509e3 1089 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1090 return -EINVAL;
1091
21d509e3 1092 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1093 return -EINVAL;
1094
1095 /* Having something in the write domain implies it's in the read
1096 * domain, and only that read domain. Enforce that in the request.
1097 */
1098 if (write_domain != 0 && read_domains != write_domain)
1099 return -EINVAL;
1100
673a394b
EA		 1101 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1102 if (obj == NULL)
bf79cb91 1103 return -ENOENT;
23010e43 1104 obj_priv = to_intel_bo(obj);
673a394b 1105
76c1dec1
CW		 1106 ret = i915_mutex_lock_interruptible(dev);
1107 if (ret) {
1108 drm_gem_object_unreference_unlocked(obj);
1109 return ret;
1110 }
652c393a
JB		 1111
1112 intel_mark_busy(dev, obj);
1113
2ef7eeaa
EA		 1114 if (read_domains & I915_GEM_DOMAIN_GTT) {
1115 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392 1116
a09ba7fa
EA		 1117 /* Update the LRU on the fence for the CPU access that's
1118 * about to occur.
1119 */
1120 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
007cc8ac
DV		 1121 struct drm_i915_fence_reg *reg =
1122 &dev_priv->fence_regs[obj_priv->fence_reg];
1123 list_move_tail(&reg->lru_list,
a09ba7fa
EA		 1124 &dev_priv->mm.fence_list);
1125 }
1126
02354392
EA		 1127 /* Silently promote "you're not bound, there was nothing to do"
1128 * to success, since the client was just asking us to
1129 * make sure everything was done.
1130 */
1131 if (ret == -EINVAL)
1132 ret = 0;
2ef7eeaa 1133 } else {
e47c68e9 1134 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1135 }
1136
7d1c4804
CW		 1137 /* Maintain LRU order of "inactive" objects */
1138 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1139 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1140
673a394b
EA		 1141 drm_gem_object_unreference(obj);
1142 mutex_unlock(&dev->struct_mutex);
1143 return ret;
1144}
1145
1146/**
1147 * Called when user space has done writes to this buffer
1148 */
1149int
1150i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1151 struct drm_file *file_priv)
1152{
1153 struct drm_i915_gem_sw_finish *args = data;
1154 struct drm_gem_object *obj;
673a394b
EA		 1155 int ret = 0;
1156
1157 if (!(dev->driver->driver_features & DRIVER_GEM))
1158 return -ENODEV;
1159
673a394b 1160 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
76c1dec1 1161 if (obj == NULL)
bf79cb91 1162 return -ENOENT;
76c1dec1
CW		 1163
1164 ret = i915_mutex_lock_interruptible(dev);
1165 if (ret) {
1166 drm_gem_object_unreference_unlocked(obj);
1167 return ret;
673a394b
EA		 1168 }
1169
673a394b 1170 /* Pinned buffers may be scanout, so flush the cache */
3d2a812a 1171 if (to_intel_bo(obj)->pin_count)
e47c68e9
EA		 1172 i915_gem_object_flush_cpu_write_domain(obj);
1173
673a394b
EA		 1174 drm_gem_object_unreference(obj);
1175 mutex_unlock(&dev->struct_mutex);
1176 return ret;
1177}
1178
1179/**
1180 * Maps the contents of an object, returning the address it is mapped
1181 * into.
1182 *
1183 * While the mapping holds a reference on the contents of the object, it doesn't
1184 * imply a ref on the object itself.
1185 */
1186int
1187i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1188 struct drm_file *file_priv)
1189{
1190 struct drm_i915_gem_mmap *args = data;
1191 struct drm_gem_object *obj;
1192 loff_t offset;
1193 unsigned long addr;
1194
1195 if (!(dev->driver->driver_features & DRIVER_GEM))
1196 return -ENODEV;
1197
1198 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1199 if (obj == NULL)
bf79cb91 1200 return -ENOENT;
673a394b
EA		 1201
1202 offset = args->offset;
1203
1204 down_write(&current->mm->mmap_sem);
1205 addr = do_mmap(obj->filp, 0, args->size,
1206 PROT_READ | PROT_WRITE, MAP_SHARED,
1207 args->offset);
1208 up_write(&current->mm->mmap_sem);
bc9025bd 1209 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1210 if (IS_ERR((void *)addr))
1211 return addr;
1212
1213 args->addr_ptr = (uint64_t) addr;
1214
1215 return 0;
1216}
1217
de151cf6
JB		 1218/**
1219 * i915_gem_fault - fault a page into the GTT
1220 * vma: VMA in question
1221 * vmf: fault info
1222 *
1223 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1224 * from userspace. The fault handler takes care of binding the object to
1225 * the GTT (if needed), allocating and programming a fence register (again,
1226 * only if needed based on whether the old reg is still valid or the object
1227 * is tiled) and inserting a new PTE into the faulting process.
1228 *
1229 * Note that the faulting process may involve evicting existing objects
1230 * from the GTT and/or fence registers to make room. So performance may
1231 * suffer if the GTT working set is large or there are few fence registers
1232 * left.
1233 */
1234int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1235{
1236 struct drm_gem_object *obj = vma->vm_private_data;
1237 struct drm_device *dev = obj->dev;
7d1c4804 1238 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1239 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 1240 pgoff_t page_offset;
1241 unsigned long pfn;
1242 int ret = 0;
0f973f27 1243 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1244
1245 /* We don't use vmf->pgoff since that has the fake offset */
1246 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1247 PAGE_SHIFT;
1248
1249 /* Now bind it into the GTT if needed */
1250 mutex_lock(&dev->struct_mutex);
1251 if (!obj_priv->gtt_space) {
e67b8ce1 1252 ret = i915_gem_object_bind_to_gtt(obj, 0);
c715089f
CW		 1253 if (ret)
1254 goto unlock;
07f4f3e8 1255
07f4f3e8 1256 ret = i915_gem_object_set_to_gtt_domain(obj, write);
c715089f
CW		 1257 if (ret)
1258 goto unlock;
de151cf6
JB		 1259 }
1260
1261 /* Need a new fence register? */
a09ba7fa 1262 if (obj_priv->tiling_mode != I915_TILING_NONE) {
2cf34d7b 1263 ret = i915_gem_object_get_fence_reg(obj, true);
c715089f
CW		 1264 if (ret)
1265 goto unlock;
d9ddcb96 1266 }
de151cf6 1267
7d1c4804
CW		 1268 if (i915_gem_object_is_inactive(obj_priv))
1269 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1270
de151cf6
JB		 1271 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1272 page_offset;
1273
1274 /* Finally, remap it using the new GTT offset */
1275 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1276unlock:
de151cf6
JB		 1277 mutex_unlock(&dev->struct_mutex);
1278
1279 switch (ret) {
c715089f
CW		 1280 case 0:
1281 case -ERESTARTSYS:
1282 return VM_FAULT_NOPAGE;
de151cf6
JB		 1283 case -ENOMEM:
1284 case -EAGAIN:
1285 return VM_FAULT_OOM;
de151cf6 1286 default:
c715089f 1287 return VM_FAULT_SIGBUS;
de151cf6
JB		 1288 }
1289}
1290
1291/**
1292 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1293 * @obj: obj in question
1294 *
1295 * GEM memory mapping works by handing back to userspace a fake mmap offset
1296 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1297 * up the object based on the offset and sets up the various memory mapping
1298 * structures.
1299 *
1300 * This routine allocates and attaches a fake offset for @obj.
1301 */
1302static int
1303i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1304{
1305 struct drm_device *dev = obj->dev;
1306 struct drm_gem_mm *mm = dev->mm_private;
23010e43 1307 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 1308 struct drm_map_list *list;
f77d390c 1309 struct drm_local_map *map;
de151cf6
JB		 1310 int ret = 0;
1311
1312 /* Set the object up for mmap'ing */
1313 list = &obj->map_list;
9a298b2a 1314 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
de151cf6
JB		 1315 if (!list->map)
1316 return -ENOMEM;
1317
1318 map = list->map;
1319 map->type = _DRM_GEM;
1320 map->size = obj->size;
1321 map->handle = obj;
1322
1323 /* Get a DRM GEM mmap offset allocated... */
1324 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1325 obj->size / PAGE_SIZE, 0, 0);
1326 if (!list->file_offset_node) {
1327 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
9e0ae534 1328 ret = -ENOSPC;
de151cf6
JB		 1329 goto out_free_list;
1330 }
1331
1332 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1333 obj->size / PAGE_SIZE, 0);
1334 if (!list->file_offset_node) {
1335 ret = -ENOMEM;
1336 goto out_free_list;
1337 }
1338
1339 list->hash.key = list->file_offset_node->start;
9e0ae534
CW		 1340 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1341 if (ret) {
de151cf6
JB		 1342 DRM_ERROR("failed to add to map hash\n");
1343 goto out_free_mm;
1344 }
1345
1346 /* By now we should be all set, any drm_mmap request on the offset
1347 * below will get to our mmap & fault handler */
1348 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1349
1350 return 0;
1351
1352out_free_mm:
1353 drm_mm_put_block(list->file_offset_node);
1354out_free_list:
9a298b2a 1355 kfree(list->map);
de151cf6
JB		 1356
1357 return ret;
1358}
1359
901782b2
CW		 1360/**
1361 * i915_gem_release_mmap - remove physical page mappings
1362 * @obj: obj in question
1363 *
af901ca1 1364 * Preserve the reservation of the mmapping with the DRM core code, but
901782b2
CW		 1365 * relinquish ownership of the pages back to the system.
1366 *
1367 * It is vital that we remove the page mapping if we have mapped a tiled
1368 * object through the GTT and then lose the fence register due to
1369 * resource pressure. Similarly if the object has been moved out of the
1370 * aperture, than pages mapped into userspace must be revoked. Removing the
1371 * mapping will then trigger a page fault on the next user access, allowing
1372 * fixup by i915_gem_fault().
1373 */
d05ca301 1374void
901782b2
CW		 1375i915_gem_release_mmap(struct drm_gem_object *obj)
1376{
1377 struct drm_device *dev = obj->dev;
23010e43 1378 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
901782b2
CW		 1379
1380 if (dev->dev_mapping)
1381 unmap_mapping_range(dev->dev_mapping,
1382 obj_priv->mmap_offset, obj->size, 1);
1383}
1384
ab00b3e5
JB		 1385static void
1386i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1387{
1388 struct drm_device *dev = obj->dev;
23010e43 1389 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ab00b3e5
JB		 1390 struct drm_gem_mm *mm = dev->mm_private;
1391 struct drm_map_list *list;
1392
1393 list = &obj->map_list;
1394 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1395
1396 if (list->file_offset_node) {
1397 drm_mm_put_block(list->file_offset_node);
1398 list->file_offset_node = NULL;
1399 }
1400
1401 if (list->map) {
9a298b2a 1402 kfree(list->map);
ab00b3e5
JB		 1403 list->map = NULL;
1404 }
1405
1406 obj_priv->mmap_offset = 0;
1407}
1408
de151cf6
JB		 1409/**
1410 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1411 * @obj: object to check
1412 *
1413 * Return the required GTT alignment for an object, taking into account
1414 * potential fence register mapping if needed.
1415 */
1416static uint32_t
1417i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1418{
1419 struct drm_device *dev = obj->dev;
23010e43 1420 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 1421 int start, i;
1422
1423 /*
1424 * Minimum alignment is 4k (GTT page size), but might be greater
1425 * if a fence register is needed for the object.
1426 */
a6c45cf0 1427 if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
de151cf6
JB		 1428 return 4096;
1429
1430 /*
1431 * Previous chips need to be aligned to the size of the smallest
1432 * fence register that can contain the object.
1433 */
a6c45cf0 1434 if (INTEL_INFO(dev)->gen == 3)
de151cf6
JB		 1435 start = 1024*1024;
1436 else
1437 start = 512*1024;
1438
1439 for (i = start; i < obj->size; i <<= 1)
1440 ;
1441
1442 return i;
1443}
1444
1445/**
1446 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1447 * @dev: DRM device
1448 * @data: GTT mapping ioctl data
1449 * @file_priv: GEM object info
1450 *
1451 * Simply returns the fake offset to userspace so it can mmap it.
1452 * The mmap call will end up in drm_gem_mmap(), which will set things
1453 * up so we can get faults in the handler above.
1454 *
1455 * The fault handler will take care of binding the object into the GTT
1456 * (since it may have been evicted to make room for something), allocating
1457 * a fence register, and mapping the appropriate aperture address into
1458 * userspace.
1459 */
1460int
1461i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1462 struct drm_file *file_priv)
1463{
1464 struct drm_i915_gem_mmap_gtt *args = data;
de151cf6
JB		 1465 struct drm_gem_object *obj;
1466 struct drm_i915_gem_object *obj_priv;
1467 int ret;
1468
1469 if (!(dev->driver->driver_features & DRIVER_GEM))
1470 return -ENODEV;
1471
1472 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1473 if (obj == NULL)
bf79cb91 1474 return -ENOENT;
de151cf6 1475
76c1dec1
CW		 1476 ret = i915_mutex_lock_interruptible(dev);
1477 if (ret) {
1478 drm_gem_object_unreference_unlocked(obj);
1479 return ret;
1480 }
de151cf6 1481
23010e43 1482 obj_priv = to_intel_bo(obj);
de151cf6 1483
ab18282d
CW		 1484 if (obj_priv->madv != I915_MADV_WILLNEED) {
1485 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1486 drm_gem_object_unreference(obj);
1487 mutex_unlock(&dev->struct_mutex);
1488 return -EINVAL;
1489 }
1490
1491
de151cf6
JB		 1492 if (!obj_priv->mmap_offset) {
1493 ret = i915_gem_create_mmap_offset(obj);
13af1062
CW		 1494 if (ret) {
1495 drm_gem_object_unreference(obj);
1496 mutex_unlock(&dev->struct_mutex);
de151cf6 1497 return ret;
13af1062 1498 }
de151cf6
JB		 1499 }
1500
1501 args->offset = obj_priv->mmap_offset;
1502
de151cf6
JB		 1503 /*
1504 * Pull it into the GTT so that we have a page list (makes the
1505 * initial fault faster and any subsequent flushing possible).
1506 */
1507 if (!obj_priv->agp_mem) {
e67b8ce1 1508 ret = i915_gem_object_bind_to_gtt(obj, 0);
de151cf6
JB		 1509 if (ret) {
1510 drm_gem_object_unreference(obj);
1511 mutex_unlock(&dev->struct_mutex);
1512 return ret;
1513 }
de151cf6
JB		 1514 }
1515
1516 drm_gem_object_unreference(obj);
1517 mutex_unlock(&dev->struct_mutex);
1518
1519 return 0;
1520}
1521
5cdf5881 1522static void
856fa198 1523i915_gem_object_put_pages(struct drm_gem_object *obj)
673a394b 1524{
23010e43 1525 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 1526 int page_count = obj->size / PAGE_SIZE;
1527 int i;
1528
856fa198 1529 BUG_ON(obj_priv->pages_refcount == 0);
bb6baf76 1530 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
673a394b 1531
856fa198
EA		 1532 if (--obj_priv->pages_refcount != 0)
1533 return;
673a394b 1534
280b713b
EA		 1535 if (obj_priv->tiling_mode != I915_TILING_NONE)
1536 i915_gem_object_save_bit_17_swizzle(obj);
1537
3ef94daa 1538 if (obj_priv->madv == I915_MADV_DONTNEED)
13a05fd9 1539 obj_priv->dirty = 0;
3ef94daa
CW		 1540
1541 for (i = 0; i < page_count; i++) {
3ef94daa
CW		 1542 if (obj_priv->dirty)
1543 set_page_dirty(obj_priv->pages[i]);
1544
1545 if (obj_priv->madv == I915_MADV_WILLNEED)
856fa198 1546 mark_page_accessed(obj_priv->pages[i]);
3ef94daa
CW		 1547
1548 page_cache_release(obj_priv->pages[i]);
1549 }
673a394b
EA		 1550 obj_priv->dirty = 0;
1551
8e7d2b2c 1552 drm_free_large(obj_priv->pages);
856fa198 1553 obj_priv->pages = NULL;
673a394b
EA		 1554}
1555
a56ba56c
CW		 1556static uint32_t
1557i915_gem_next_request_seqno(struct drm_device *dev,
1558 struct intel_ring_buffer *ring)
1559{
1560 drm_i915_private_t *dev_priv = dev->dev_private;
1561
1562 ring->outstanding_lazy_request = true;
1563 return dev_priv->next_seqno;
1564}
1565
673a394b 1566static void
617dbe27 1567i915_gem_object_move_to_active(struct drm_gem_object *obj,
852835f3 1568 struct intel_ring_buffer *ring)
673a394b 1569{
a56ba56c 1570 struct drm_device *dev = obj->dev;
23010e43 1571 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
a56ba56c 1572 uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
617dbe27 1573
852835f3
ZN		 1574 BUG_ON(ring == NULL);
1575 obj_priv->ring = ring;
673a394b
EA		 1576
1577 /* Add a reference if we're newly entering the active list. */
1578 if (!obj_priv->active) {
1579 drm_gem_object_reference(obj);
1580 obj_priv->active = 1;
1581 }
e35a41de 1582
673a394b 1583 /* Move from whatever list we were on to the tail of execution. */
852835f3 1584 list_move_tail(&obj_priv->list, &ring->active_list);
a56ba56c 1585 obj_priv->last_rendering_seqno = seqno;
673a394b
EA		 1586}
1587
ce44b0ea
EA		 1588static void
1589i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1590{
1591 struct drm_device *dev = obj->dev;
1592 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1593 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ce44b0ea
EA		 1594
1595 BUG_ON(!obj_priv->active);
1596 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1597 obj_priv->last_rendering_seqno = 0;
1598}
673a394b 1599
963b4836
CW		 1600/* Immediately discard the backing storage */
1601static void
1602i915_gem_object_truncate(struct drm_gem_object *obj)
1603{
23010e43 1604 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
bb6baf76 1605 struct inode *inode;
963b4836 1606
ae9fed6b
CW		 1607 /* Our goal here is to return as much of the memory as
1608 * is possible back to the system as we are called from OOM.
1609 * To do this we must instruct the shmfs to drop all of its
1610 * backing pages, *now*. Here we mirror the actions taken
1611 * when by shmem_delete_inode() to release the backing store.
1612 */
bb6baf76 1613 inode = obj->filp->f_path.dentry->d_inode;
ae9fed6b
CW		 1614 truncate_inode_pages(inode->i_mapping, 0);
1615 if (inode->i_op->truncate_range)
1616 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
bb6baf76
CW		 1617
1618 obj_priv->madv = __I915_MADV_PURGED;
963b4836
CW		 1619}
1620
1621static inline int
1622i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1623{
1624 return obj_priv->madv == I915_MADV_DONTNEED;
1625}
1626
673a394b
EA		 1627static void
1628i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1629{
1630 struct drm_device *dev = obj->dev;
1631 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1632 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 1633
673a394b 1634 if (obj_priv->pin_count != 0)
f13d3f73 1635 list_move_tail(&obj_priv->list, &dev_priv->mm.pinned_list);
673a394b
EA		 1636 else
1637 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1638
99fcb766
DV		 1639 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1640
ce44b0ea 1641 obj_priv->last_rendering_seqno = 0;
852835f3 1642 obj_priv->ring = NULL;
673a394b
EA		 1643 if (obj_priv->active) {
1644 obj_priv->active = 0;
1645 drm_gem_object_unreference(obj);
1646 }
23bc5982 1647 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 1648}
1649
9220434a 1650static void
63560396 1651i915_gem_process_flushing_list(struct drm_device *dev,
8a1a49f9 1652 uint32_t flush_domains,
852835f3 1653 struct intel_ring_buffer *ring)
63560396
DV		 1654{
1655 drm_i915_private_t *dev_priv = dev->dev_private;
1656 struct drm_i915_gem_object *obj_priv, *next;
1657
1658 list_for_each_entry_safe(obj_priv, next,
1659 &dev_priv->mm.gpu_write_list,
1660 gpu_write_list) {
a8089e84 1661 struct drm_gem_object *obj = &obj_priv->base;
63560396 1662
2b6efaa4
CW		 1663 if (obj->write_domain & flush_domains &&
1664 obj_priv->ring == ring) {
63560396
DV		 1665 uint32_t old_write_domain = obj->write_domain;
1666
1667 obj->write_domain = 0;
1668 list_del_init(&obj_priv->gpu_write_list);
617dbe27 1669 i915_gem_object_move_to_active(obj, ring);
63560396
DV		 1670
1671 /* update the fence lru list */
007cc8ac
DV		 1672 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1673 struct drm_i915_fence_reg *reg =
1674 &dev_priv->fence_regs[obj_priv->fence_reg];
1675 list_move_tail(&reg->lru_list,
63560396 1676 &dev_priv->mm.fence_list);
007cc8ac 1677 }
63560396
DV		 1678
1679 trace_i915_gem_object_change_domain(obj,
1680 obj->read_domains,
1681 old_write_domain);
1682 }
1683 }
1684}
8187a2b7 1685
5a5a0c64 1686uint32_t
8a1a49f9 1687i915_add_request(struct drm_device *dev,
f787a5f5 1688 struct drm_file *file,
8dc5d147 1689 struct drm_i915_gem_request *request,
8a1a49f9 1690 struct intel_ring_buffer *ring)
673a394b
EA		 1691{
1692 drm_i915_private_t *dev_priv = dev->dev_private;
f787a5f5 1693 struct drm_i915_file_private *file_priv = NULL;
673a394b
EA		 1694 uint32_t seqno;
1695 int was_empty;
673a394b 1696
f787a5f5
CW		 1697 if (file != NULL)
1698 file_priv = file->driver_priv;
b962442e 1699
8dc5d147
CW		 1700 if (request == NULL) {
1701 request = kzalloc(sizeof(*request), GFP_KERNEL);
1702 if (request == NULL)
1703 return 0;
1704 }
673a394b 1705
f787a5f5 1706 seqno = ring->add_request(dev, ring, 0);
a56ba56c 1707 ring->outstanding_lazy_request = false;
673a394b
EA		 1708
1709 request->seqno = seqno;
852835f3 1710 request->ring = ring;
673a394b 1711 request->emitted_jiffies = jiffies;
852835f3
ZN		 1712 was_empty = list_empty(&ring->request_list);
1713 list_add_tail(&request->list, &ring->request_list);
1714
f787a5f5 1715 if (file_priv) {
1c25595f 1716 spin_lock(&file_priv->mm.lock);
f787a5f5 1717 request->file_priv = file_priv;
b962442e 1718 list_add_tail(&request->client_list,
f787a5f5 1719 &file_priv->mm.request_list);
1c25595f 1720 spin_unlock(&file_priv->mm.lock);
b962442e 1721 }
673a394b 1722
f65d9421 1723 if (!dev_priv->mm.suspended) {
b3b079db
CW		 1724 mod_timer(&dev_priv->hangcheck_timer,
1725 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
f65d9421 1726 if (was_empty)
b3b079db
CW		 1727 queue_delayed_work(dev_priv->wq,
1728 &dev_priv->mm.retire_work, HZ);
f65d9421 1729 }
673a394b
EA		 1730 return seqno;
1731}
1732
1733/**
1734 * Command execution barrier
1735 *
1736 * Ensures that all commands in the ring are finished
1737 * before signalling the CPU
1738 */
8a1a49f9 1739static void
852835f3 1740i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
673a394b 1741{
673a394b 1742 uint32_t flush_domains = 0;
673a394b
EA		 1743
1744 /* The sampler always gets flushed on i965 (sigh) */
a6c45cf0 1745 if (INTEL_INFO(dev)->gen >= 4)
673a394b 1746 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
852835f3
ZN		 1747
1748 ring->flush(dev, ring,
1749 I915_GEM_DOMAIN_COMMAND, flush_domains);
673a394b
EA		 1750}
1751
f787a5f5
CW		 1752static inline void
1753i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
673a394b 1754{
1c25595f
CW		 1755 struct drm_i915_file_private *file_priv = request->file_priv;
1756
1757 if (!file_priv)
1758 return;
1759
1760 spin_lock(&file_priv->mm.lock);
1761 list_del(&request->client_list);
1762 request->file_priv = NULL;
1763 spin_unlock(&file_priv->mm.lock);
673a394b
EA		 1764}
1765
dfaae392
CW		 1766static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1767 struct intel_ring_buffer *ring)
9375e446 1768{
dfaae392
CW		 1769 while (!list_empty(&ring->request_list)) {
1770 struct drm_i915_gem_request *request;
9375e446 1771
dfaae392
CW		 1772 request = list_first_entry(&ring->request_list,
1773 struct drm_i915_gem_request,
1774 list);
1775
1776 list_del(&request->list);
f787a5f5 1777 i915_gem_request_remove_from_client(request);
dfaae392
CW		 1778 kfree(request);
1779 }
1780
1781 while (!list_empty(&ring->active_list)) {
9375e446
CW		 1782 struct drm_i915_gem_object *obj_priv;
1783
dfaae392 1784 obj_priv = list_first_entry(&ring->active_list,
9375e446
CW		 1785 struct drm_i915_gem_object,
1786 list);
1787
1788 obj_priv->base.write_domain = 0;
dfaae392 1789 list_del_init(&obj_priv->gpu_write_list);
9375e446
CW		 1790 i915_gem_object_move_to_inactive(&obj_priv->base);
1791 }
1792}
1793
069efc1d 1794void i915_gem_reset(struct drm_device *dev)
77f01230
CW		 1795{
1796 struct drm_i915_private *dev_priv = dev->dev_private;
1797 struct drm_i915_gem_object *obj_priv;
069efc1d 1798 int i;
77f01230 1799
dfaae392
CW		 1800 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1801 if (HAS_BSD(dev))
1802 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1803
1804 /* Remove anything from the flushing lists. The GPU cache is likely
1805 * to be lost on reset along with the data, so simply move the
1806 * lost bo to the inactive list.
1807 */
1808 while (!list_empty(&dev_priv->mm.flushing_list)) {
1809 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1810 struct drm_i915_gem_object,
1811 list);
1812
1813 obj_priv->base.write_domain = 0;
1814 list_del_init(&obj_priv->gpu_write_list);
1815 i915_gem_object_move_to_inactive(&obj_priv->base);
1816 }
1817
1818 /* Move everything out of the GPU domains to ensure we do any
1819 * necessary invalidation upon reuse.
1820 */
77f01230
CW		 1821 list_for_each_entry(obj_priv,
1822 &dev_priv->mm.inactive_list,
1823 list)
1824 {
1825 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1826 }
069efc1d
CW		 1827
1828 /* The fence registers are invalidated so clear them out */
1829 for (i = 0; i < 16; i++) {
1830 struct drm_i915_fence_reg *reg;
1831
1832 reg = &dev_priv->fence_regs[i];
1833 if (!reg->obj)
1834 continue;
1835
1836 i915_gem_clear_fence_reg(reg->obj);
1837 }
77f01230
CW		 1838}
1839
673a394b
EA		 1840/**
1841 * This function clears the request list as sequence numbers are passed.
1842 */
b09a1fec
CW		 1843static void
1844i915_gem_retire_requests_ring(struct drm_device *dev,
1845 struct intel_ring_buffer *ring)
673a394b
EA		 1846{
1847 drm_i915_private_t *dev_priv = dev->dev_private;
1848 uint32_t seqno;
1849
b84d5f0c
CW		 1850 if (!ring->status_page.page_addr ||
1851 list_empty(&ring->request_list))
6c0594a3
KW		 1852 return;
1853
23bc5982
CW		 1854 WARN_ON(i915_verify_lists(dev));
1855
f787a5f5 1856 seqno = ring->get_seqno(dev, ring);
852835f3 1857 while (!list_empty(&ring->request_list)) {
673a394b 1858 struct drm_i915_gem_request *request;
673a394b 1859
852835f3 1860 request = list_first_entry(&ring->request_list,
673a394b
EA		 1861 struct drm_i915_gem_request,
1862 list);
673a394b 1863
dfaae392 1864 if (!i915_seqno_passed(seqno, request->seqno))
b84d5f0c
CW		 1865 break;
1866
1867 trace_i915_gem_request_retire(dev, request->seqno);
1868
1869 list_del(&request->list);
f787a5f5 1870 i915_gem_request_remove_from_client(request);
b84d5f0c
CW		 1871 kfree(request);
1872 }
1873
1874 /* Move any buffers on the active list that are no longer referenced
1875 * by the ringbuffer to the flushing/inactive lists as appropriate.
1876 */
1877 while (!list_empty(&ring->active_list)) {
1878 struct drm_gem_object *obj;
1879 struct drm_i915_gem_object *obj_priv;
1880
1881 obj_priv = list_first_entry(&ring->active_list,
1882 struct drm_i915_gem_object,
1883 list);
673a394b 1884
dfaae392 1885 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
673a394b 1886 break;
b84d5f0c
CW		 1887
1888 obj = &obj_priv->base;
b84d5f0c
CW		 1889 if (obj->write_domain != 0)
1890 i915_gem_object_move_to_flushing(obj);
1891 else
1892 i915_gem_object_move_to_inactive(obj);
673a394b 1893 }
9d34e5db
CW		 1894
1895 if (unlikely (dev_priv->trace_irq_seqno &&
1896 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
8187a2b7 1897 ring->user_irq_put(dev, ring);
9d34e5db
CW		 1898 dev_priv->trace_irq_seqno = 0;
1899 }
23bc5982
CW		 1900
1901 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 1902}
1903
b09a1fec
CW		 1904void
1905i915_gem_retire_requests(struct drm_device *dev)
1906{
1907 drm_i915_private_t *dev_priv = dev->dev_private;
1908
be72615b
CW		 1909 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1910 struct drm_i915_gem_object *obj_priv, *tmp;
1911
1912 /* We must be careful that during unbind() we do not
1913 * accidentally infinitely recurse into retire requests.
1914 * Currently:
1915 * retire -> free -> unbind -> wait -> retire_ring
1916 */
1917 list_for_each_entry_safe(obj_priv, tmp,
1918 &dev_priv->mm.deferred_free_list,
1919 list)
1920 i915_gem_free_object_tail(&obj_priv->base);
1921 }
1922
b09a1fec
CW		 1923 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1924 if (HAS_BSD(dev))
1925 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1926}
1927
75ef9da2 1928static void
673a394b
EA		 1929i915_gem_retire_work_handler(struct work_struct *work)
1930{
1931 drm_i915_private_t *dev_priv;
1932 struct drm_device *dev;
1933
1934 dev_priv = container_of(work, drm_i915_private_t,
1935 mm.retire_work.work);
1936 dev = dev_priv->dev;
1937
891b48cf
CW		 1938 /* Come back later if the device is busy... */
1939 if (!mutex_trylock(&dev->struct_mutex)) {
1940 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1941 return;
1942 }
1943
b09a1fec 1944 i915_gem_retire_requests(dev);
d1b851fc 1945
6dbe2772 1946 if (!dev_priv->mm.suspended &&
d1b851fc
ZN		 1947 (!list_empty(&dev_priv->render_ring.request_list) ||
1948 (HAS_BSD(dev) &&
1949 !list_empty(&dev_priv->bsd_ring.request_list))))
9c9fe1f8 1950 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
673a394b
EA		 1951 mutex_unlock(&dev->struct_mutex);
1952}
1953
5a5a0c64 1954int
852835f3 1955i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
8a1a49f9 1956 bool interruptible, struct intel_ring_buffer *ring)
673a394b
EA		 1957{
1958 drm_i915_private_t *dev_priv = dev->dev_private;
802c7eb6 1959 u32 ier;
673a394b
EA		 1960 int ret = 0;
1961
1962 BUG_ON(seqno == 0);
1963
30dbf0c0
CW		 1964 if (atomic_read(&dev_priv->mm.wedged))
1965 return -EAGAIN;
1966
a56ba56c 1967 if (ring->outstanding_lazy_request) {
8dc5d147 1968 seqno = i915_add_request(dev, NULL, NULL, ring);
e35a41de
DV		 1969 if (seqno == 0)
1970 return -ENOMEM;
1971 }
a56ba56c 1972 BUG_ON(seqno == dev_priv->next_seqno);
e35a41de 1973
f787a5f5 1974 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
bad720ff 1975 if (HAS_PCH_SPLIT(dev))
036a4a7d
ZW		 1976 ier = I915_READ(DEIER) | I915_READ(GTIER);
1977 else
1978 ier = I915_READ(IER);
802c7eb6
JB		 1979 if (!ier) {
1980 DRM_ERROR("something (likely vbetool) disabled "
1981 "interrupts, re-enabling\n");
1982 i915_driver_irq_preinstall(dev);
1983 i915_driver_irq_postinstall(dev);
1984 }
1985
1c5d22f7
CW		 1986 trace_i915_gem_request_wait_begin(dev, seqno);
1987
852835f3 1988 ring->waiting_gem_seqno = seqno;
8187a2b7 1989 ring->user_irq_get(dev, ring);
48764bf4 1990 if (interruptible)
852835f3
ZN		 1991 ret = wait_event_interruptible(ring->irq_queue,
1992 i915_seqno_passed(
f787a5f5 1993 ring->get_seqno(dev, ring), seqno)
852835f3 1994 || atomic_read(&dev_priv->mm.wedged));
48764bf4 1995 else
852835f3
ZN		 1996 wait_event(ring->irq_queue,
1997 i915_seqno_passed(
f787a5f5 1998 ring->get_seqno(dev, ring), seqno)
852835f3 1999 || atomic_read(&dev_priv->mm.wedged));
48764bf4 2000
8187a2b7 2001 ring->user_irq_put(dev, ring);
852835f3 2002 ring->waiting_gem_seqno = 0;
1c5d22f7
CW		 2003
2004 trace_i915_gem_request_wait_end(dev, seqno);
673a394b 2005 }
ba1234d1 2006 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0 2007 ret = -EAGAIN;
673a394b
EA		 2008
2009 if (ret && ret != -ERESTARTSYS)
8bff917c 2010 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
f787a5f5 2011 __func__, ret, seqno, ring->get_seqno(dev, ring),
8bff917c 2012 dev_priv->next_seqno);
673a394b
EA		 2013
2014 /* Directly dispatch request retiring. While we have the work queue
2015 * to handle this, the waiter on a request often wants an associated
2016 * buffer to have made it to the inactive list, and we would need
2017 * a separate wait queue to handle that.
2018 */
2019 if (ret == 0)
b09a1fec 2020 i915_gem_retire_requests_ring(dev, ring);
673a394b
EA		 2021
2022 return ret;
2023}
2024
48764bf4
DV		 2025/**
2026 * Waits for a sequence number to be signaled, and cleans up the
2027 * request and object lists appropriately for that event.
2028 */
2029static int
852835f3 2030i915_wait_request(struct drm_device *dev, uint32_t seqno,
a56ba56c 2031 struct intel_ring_buffer *ring)
48764bf4 2032{
852835f3 2033 return i915_do_wait_request(dev, seqno, 1, ring);
48764bf4
DV		 2034}
2035
20f0cd55 2036static void
9220434a 2037i915_gem_flush_ring(struct drm_device *dev,
c78ec30b 2038 struct drm_file *file_priv,
9220434a
CW		 2039 struct intel_ring_buffer *ring,
2040 uint32_t invalidate_domains,
2041 uint32_t flush_domains)
2042{
2043 ring->flush(dev, ring, invalidate_domains, flush_domains);
2044 i915_gem_process_flushing_list(dev, flush_domains, ring);
2045}
2046
8187a2b7
ZN		 2047static void
2048i915_gem_flush(struct drm_device *dev,
c78ec30b 2049 struct drm_file *file_priv,
8187a2b7 2050 uint32_t invalidate_domains,
9220434a
CW		 2051 uint32_t flush_domains,
2052 uint32_t flush_rings)
8187a2b7
ZN		 2053{
2054 drm_i915_private_t *dev_priv = dev->dev_private;
8bff917c 2055
8187a2b7
ZN		 2056 if (flush_domains & I915_GEM_DOMAIN_CPU)
2057 drm_agp_chipset_flush(dev);
8bff917c 2058
9220434a
CW		 2059 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2060 if (flush_rings & RING_RENDER)
c78ec30b 2061 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 2062 &dev_priv->render_ring,
2063 invalidate_domains, flush_domains);
2064 if (flush_rings & RING_BSD)
c78ec30b 2065 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 2066 &dev_priv->bsd_ring,
2067 invalidate_domains, flush_domains);
2068 }
8187a2b7
ZN		 2069}
2070
673a394b
EA		 2071/**
2072 * Ensures that all rendering to the object has completed and the object is
2073 * safe to unbind from the GTT or access from the CPU.
2074 */
2075static int
2cf34d7b
CW		 2076i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2077 bool interruptible)
673a394b
EA		 2078{
2079 struct drm_device *dev = obj->dev;
23010e43 2080 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2081 int ret;
2082
e47c68e9
EA		 2083 /* This function only exists to support waiting for existing rendering,
2084 * not for emitting required flushes.
673a394b 2085 */
e47c68e9 2086 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 2087
2088 /* If there is rendering queued on the buffer being evicted, wait for
2089 * it.
2090 */
2091 if (obj_priv->active) {
2cf34d7b
CW		 2092 ret = i915_do_wait_request(dev,
2093 obj_priv->last_rendering_seqno,
2094 interruptible,
2095 obj_priv->ring);
2096 if (ret)
673a394b
EA		 2097 return ret;
2098 }
2099
2100 return 0;
2101}
2102
2103/**
2104 * Unbinds an object from the GTT aperture.
2105 */
0f973f27 2106int
673a394b
EA		 2107i915_gem_object_unbind(struct drm_gem_object *obj)
2108{
2109 struct drm_device *dev = obj->dev;
73aa808f 2110 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2111 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2112 int ret = 0;
2113
673a394b
EA		 2114 if (obj_priv->gtt_space == NULL)
2115 return 0;
2116
2117 if (obj_priv->pin_count != 0) {
2118 DRM_ERROR("Attempting to unbind pinned buffer\n");
2119 return -EINVAL;
2120 }
2121
5323fd04
EA		 2122 /* blow away mappings if mapped through GTT */
2123 i915_gem_release_mmap(obj);
2124
673a394b
EA		 2125 /* Move the object to the CPU domain to ensure that
2126 * any possible CPU writes while it's not in the GTT
2127 * are flushed when we go to remap it. This will
2128 * also ensure that all pending GPU writes are finished
2129 * before we unbind.
2130 */
e47c68e9 2131 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
8dc1775d 2132 if (ret == -ERESTARTSYS)
673a394b 2133 return ret;
8dc1775d
CW		 2134 /* Continue on if we fail due to EIO, the GPU is hung so we
2135 * should be safe and we need to cleanup or else we might
2136 * cause memory corruption through use-after-free.
2137 */
812ed492
CW		 2138 if (ret) {
2139 i915_gem_clflush_object(obj);
2140 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
2141 }
673a394b 2142
96b47b65
DV		 2143 /* release the fence reg _after_ flushing */
2144 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2145 i915_gem_clear_fence_reg(obj);
2146
73aa808f
CW		 2147 drm_unbind_agp(obj_priv->agp_mem);
2148 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
673a394b 2149
856fa198 2150 i915_gem_object_put_pages(obj);
a32808c0 2151 BUG_ON(obj_priv->pages_refcount);
673a394b 2152
73aa808f 2153 i915_gem_info_remove_gtt(dev_priv, obj->size);
f13d3f73 2154 list_del_init(&obj_priv->list);
673a394b 2155
73aa808f
CW		 2156 drm_mm_put_block(obj_priv->gtt_space);
2157 obj_priv->gtt_space = NULL;
2158
963b4836
CW		 2159 if (i915_gem_object_is_purgeable(obj_priv))
2160 i915_gem_object_truncate(obj);
2161
1c5d22f7
CW		 2162 trace_i915_gem_object_unbind(obj);
2163
8dc1775d 2164 return ret;
673a394b
EA		 2165}
2166
a56ba56c
CW		 2167static int i915_ring_idle(struct drm_device *dev,
2168 struct intel_ring_buffer *ring)
2169{
2170 i915_gem_flush_ring(dev, NULL, ring,
2171 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2172 return i915_wait_request(dev,
2173 i915_gem_next_request_seqno(dev, ring),
2174 ring);
2175}
2176
b47eb4a2 2177int
4df2faf4
DV		 2178i915_gpu_idle(struct drm_device *dev)
2179{
2180 drm_i915_private_t *dev_priv = dev->dev_private;
2181 bool lists_empty;
852835f3 2182 int ret;
4df2faf4 2183
d1b851fc
ZN		 2184 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2185 list_empty(&dev_priv->render_ring.active_list) &&
2186 (!HAS_BSD(dev) ||
2187 list_empty(&dev_priv->bsd_ring.active_list)));
4df2faf4
DV		 2188 if (lists_empty)
2189 return 0;
2190
2191 /* Flush everything onto the inactive list. */
a56ba56c 2192 ret = i915_ring_idle(dev, &dev_priv->render_ring);
8a1a49f9
DV		 2193 if (ret)
2194 return ret;
d1b851fc
ZN		 2195
2196 if (HAS_BSD(dev)) {
a56ba56c 2197 ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
d1b851fc
ZN		 2198 if (ret)
2199 return ret;
2200 }
2201
8a1a49f9 2202 return 0;
4df2faf4
DV		 2203}
2204
5cdf5881 2205static int
4bdadb97
CW		 2206i915_gem_object_get_pages(struct drm_gem_object *obj,
2207 gfp_t gfpmask)
673a394b 2208{
23010e43 2209 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2210 int page_count, i;
2211 struct address_space *mapping;
2212 struct inode *inode;
2213 struct page *page;
673a394b 2214
778c3544
DV		 2215 BUG_ON(obj_priv->pages_refcount
2216 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2217
856fa198 2218 if (obj_priv->pages_refcount++ != 0)
673a394b
EA		 2219 return 0;
2220
2221 /* Get the list of pages out of our struct file. They'll be pinned
2222 * at this point until we release them.
2223 */
2224 page_count = obj->size / PAGE_SIZE;
856fa198 2225 BUG_ON(obj_priv->pages != NULL);
8e7d2b2c 2226 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
856fa198 2227 if (obj_priv->pages == NULL) {
856fa198 2228 obj_priv->pages_refcount--;
673a394b
EA		 2229 return -ENOMEM;
2230 }
2231
2232 inode = obj->filp->f_path.dentry->d_inode;
2233 mapping = inode->i_mapping;
2234 for (i = 0; i < page_count; i++) {
4bdadb97 2235 page = read_cache_page_gfp(mapping, i,
985b823b 2236 GFP_HIGHUSER |
4bdadb97 2237 __GFP_COLD |
cd9f040d 2238 __GFP_RECLAIMABLE |
4bdadb97 2239 gfpmask);
1f2b1013
CW		 2240 if (IS_ERR(page))
2241 goto err_pages;
2242
856fa198 2243 obj_priv->pages[i] = page;
673a394b 2244 }
280b713b
EA		 2245
2246 if (obj_priv->tiling_mode != I915_TILING_NONE)
2247 i915_gem_object_do_bit_17_swizzle(obj);
2248
673a394b 2249 return 0;
1f2b1013
CW		 2250
2251err_pages:
2252 while (i--)
2253 page_cache_release(obj_priv->pages[i]);
2254
2255 drm_free_large(obj_priv->pages);
2256 obj_priv->pages = NULL;
2257 obj_priv->pages_refcount--;
2258 return PTR_ERR(page);
673a394b
EA		 2259}
2260
4e901fdc
EA		 2261static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2262{
2263 struct drm_gem_object *obj = reg->obj;
2264 struct drm_device *dev = obj->dev;
2265 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2266 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4e901fdc
EA		 2267 int regnum = obj_priv->fence_reg;
2268 uint64_t val;
2269
2270 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2271 0xfffff000) << 32;
2272 val |= obj_priv->gtt_offset & 0xfffff000;
2273 val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2274 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2275
2276 if (obj_priv->tiling_mode == I915_TILING_Y)
2277 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2278 val |= I965_FENCE_REG_VALID;
2279
2280 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2281}
2282
de151cf6
JB		 2283static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2284{
2285 struct drm_gem_object *obj = reg->obj;
2286 struct drm_device *dev = obj->dev;
2287 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2288 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 2289 int regnum = obj_priv->fence_reg;
2290 uint64_t val;
2291
2292 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2293 0xfffff000) << 32;
2294 val |= obj_priv->gtt_offset & 0xfffff000;
2295 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2296 if (obj_priv->tiling_mode == I915_TILING_Y)
2297 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2298 val |= I965_FENCE_REG_VALID;
2299
2300 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2301}
2302
2303static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2304{
2305 struct drm_gem_object *obj = reg->obj;
2306 struct drm_device *dev = obj->dev;
2307 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2308 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2309 int regnum = obj_priv->fence_reg;
0f973f27 2310 int tile_width;
dc529a4f 2311 uint32_t fence_reg, val;
de151cf6
JB		 2312 uint32_t pitch_val;
2313
2314 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2315 (obj_priv->gtt_offset & (obj->size - 1))) {
f06da264 2316 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
0f973f27 2317 __func__, obj_priv->gtt_offset, obj->size);
de151cf6
JB		 2318 return;
2319 }
2320
0f973f27
JB		 2321 if (obj_priv->tiling_mode == I915_TILING_Y &&
2322 HAS_128_BYTE_Y_TILING(dev))
2323 tile_width = 128;
de151cf6 2324 else
0f973f27
JB		 2325 tile_width = 512;
2326
2327 /* Note: pitch better be a power of two tile widths */
2328 pitch_val = obj_priv->stride / tile_width;
2329 pitch_val = ffs(pitch_val) - 1;
de151cf6 2330
c36a2a6d
DV		 2331 if (obj_priv->tiling_mode == I915_TILING_Y &&
2332 HAS_128_BYTE_Y_TILING(dev))
2333 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2334 else
2335 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2336
de151cf6
JB		 2337 val = obj_priv->gtt_offset;
2338 if (obj_priv->tiling_mode == I915_TILING_Y)
2339 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2340 val |= I915_FENCE_SIZE_BITS(obj->size);
2341 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2342 val |= I830_FENCE_REG_VALID;
2343
dc529a4f
EA		 2344 if (regnum < 8)
2345 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2346 else
2347 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2348 I915_WRITE(fence_reg, val);
de151cf6
JB		 2349}
2350
2351static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2352{
2353 struct drm_gem_object *obj = reg->obj;
2354 struct drm_device *dev = obj->dev;
2355 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2356 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 2357 int regnum = obj_priv->fence_reg;
2358 uint32_t val;
2359 uint32_t pitch_val;
8d7773a3 2360 uint32_t fence_size_bits;
de151cf6 2361
8d7773a3 2362 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
de151cf6 2363 (obj_priv->gtt_offset & (obj->size - 1))) {
8d7773a3 2364 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
0f973f27 2365 __func__, obj_priv->gtt_offset);
de151cf6
JB		 2366 return;
2367 }
2368
e76a16de
EA		 2369 pitch_val = obj_priv->stride / 128;
2370 pitch_val = ffs(pitch_val) - 1;
2371 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2372
de151cf6
JB		 2373 val = obj_priv->gtt_offset;
2374 if (obj_priv->tiling_mode == I915_TILING_Y)
2375 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
8d7773a3
DV		 2376 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2377 WARN_ON(fence_size_bits & ~0x00000f00);
2378 val |= fence_size_bits;
de151cf6
JB		 2379 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2380 val |= I830_FENCE_REG_VALID;
2381
2382 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
de151cf6
JB		 2383}
2384
2cf34d7b
CW		 2385static int i915_find_fence_reg(struct drm_device *dev,
2386 bool interruptible)
ae3db24a
DV		 2387{
2388 struct drm_i915_fence_reg *reg = NULL;
2389 struct drm_i915_gem_object *obj_priv = NULL;
2390 struct drm_i915_private *dev_priv = dev->dev_private;
2391 struct drm_gem_object *obj = NULL;
2392 int i, avail, ret;
2393
2394 /* First try to find a free reg */
2395 avail = 0;
2396 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2397 reg = &dev_priv->fence_regs[i];
2398 if (!reg->obj)
2399 return i;
2400
23010e43 2401 obj_priv = to_intel_bo(reg->obj);
ae3db24a
DV		 2402 if (!obj_priv->pin_count)
2403 avail++;
2404 }
2405
2406 if (avail == 0)
2407 return -ENOSPC;
2408
2409 /* None available, try to steal one or wait for a user to finish */
2410 i = I915_FENCE_REG_NONE;
007cc8ac
DV		 2411 list_for_each_entry(reg, &dev_priv->mm.fence_list,
2412 lru_list) {
2413 obj = reg->obj;
2414 obj_priv = to_intel_bo(obj);
ae3db24a
DV		 2415
2416 if (obj_priv->pin_count)
2417 continue;
2418
2419 /* found one! */
2420 i = obj_priv->fence_reg;
2421 break;
2422 }
2423
2424 BUG_ON(i == I915_FENCE_REG_NONE);
2425
2426 /* We only have a reference on obj from the active list. put_fence_reg
2427 * might drop that one, causing a use-after-free in it. So hold a
2428 * private reference to obj like the other callers of put_fence_reg
2429 * (set_tiling ioctl) do. */
2430 drm_gem_object_reference(obj);
2cf34d7b 2431 ret = i915_gem_object_put_fence_reg(obj, interruptible);
ae3db24a
DV		 2432 drm_gem_object_unreference(obj);
2433 if (ret != 0)
2434 return ret;
2435
2436 return i;
2437}
2438
de151cf6
JB		 2439/**
2440 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2441 * @obj: object to map through a fence reg
2442 *
2443 * When mapping objects through the GTT, userspace wants to be able to write
2444 * to them without having to worry about swizzling if the object is tiled.
2445 *
2446 * This function walks the fence regs looking for a free one for @obj,
2447 * stealing one if it can't find any.
2448 *
2449 * It then sets up the reg based on the object's properties: address, pitch
2450 * and tiling format.
2451 */
8c4b8c3f 2452int
2cf34d7b
CW		 2453i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2454 bool interruptible)
de151cf6
JB		 2455{
2456 struct drm_device *dev = obj->dev;
79e53945 2457 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2458 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2459 struct drm_i915_fence_reg *reg = NULL;
ae3db24a 2460 int ret;
de151cf6 2461
a09ba7fa
EA		 2462 /* Just update our place in the LRU if our fence is getting used. */
2463 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
007cc8ac
DV		 2464 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2465 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
a09ba7fa
EA		 2466 return 0;
2467 }
2468
de151cf6
JB		 2469 switch (obj_priv->tiling_mode) {
2470 case I915_TILING_NONE:
2471 WARN(1, "allocating a fence for non-tiled object?\n");
2472 break;
2473 case I915_TILING_X:
0f973f27
JB		 2474 if (!obj_priv->stride)
2475 return -EINVAL;
2476 WARN((obj_priv->stride & (512 - 1)),
2477 "object 0x%08x is X tiled but has non-512B pitch\n",
2478 obj_priv->gtt_offset);
de151cf6
JB		 2479 break;
2480 case I915_TILING_Y:
0f973f27
JB		 2481 if (!obj_priv->stride)
2482 return -EINVAL;
2483 WARN((obj_priv->stride & (128 - 1)),
2484 "object 0x%08x is Y tiled but has non-128B pitch\n",
2485 obj_priv->gtt_offset);
de151cf6
JB		 2486 break;
2487 }
2488
2cf34d7b 2489 ret = i915_find_fence_reg(dev, interruptible);
ae3db24a
DV		 2490 if (ret < 0)
2491 return ret;
de151cf6 2492
ae3db24a
DV		 2493 obj_priv->fence_reg = ret;
2494 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
007cc8ac 2495 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
a09ba7fa 2496
de151cf6
JB		 2497 reg->obj = obj;
2498
e259befd
CW		 2499 switch (INTEL_INFO(dev)->gen) {
2500 case 6:
4e901fdc 2501 sandybridge_write_fence_reg(reg);
e259befd
CW		 2502 break;
2503 case 5:
2504 case 4:
de151cf6 2505 i965_write_fence_reg(reg);
e259befd
CW		 2506 break;
2507 case 3:
de151cf6 2508 i915_write_fence_reg(reg);
e259befd
CW		 2509 break;
2510 case 2:
de151cf6 2511 i830_write_fence_reg(reg);
e259befd
CW		 2512 break;
2513 }
d9ddcb96 2514
ae3db24a
DV		 2515 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2516 obj_priv->tiling_mode);
1c5d22f7 2517
d9ddcb96 2518 return 0;
de151cf6
JB		 2519}
2520
2521/**
2522 * i915_gem_clear_fence_reg - clear out fence register info
2523 * @obj: object to clear
2524 *
2525 * Zeroes out the fence register itself and clears out the associated
2526 * data structures in dev_priv and obj_priv.
2527 */
2528static void
2529i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2530{
2531 struct drm_device *dev = obj->dev;
79e53945 2532 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2533 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
007cc8ac
DV		 2534 struct drm_i915_fence_reg *reg =
2535 &dev_priv->fence_regs[obj_priv->fence_reg];
e259befd 2536 uint32_t fence_reg;
de151cf6 2537
e259befd
CW		 2538 switch (INTEL_INFO(dev)->gen) {
2539 case 6:
4e901fdc
EA		 2540 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2541 (obj_priv->fence_reg * 8), 0);
e259befd
CW		 2542 break;
2543 case 5:
2544 case 4:
de151cf6 2545 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
e259befd
CW		 2546 break;
2547 case 3:
9b74f734 2548 if (obj_priv->fence_reg >= 8)
e259befd 2549 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
dc529a4f 2550 else
e259befd
CW		 2551 case 2:
2552 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
dc529a4f
EA		 2553
2554 I915_WRITE(fence_reg, 0);
e259befd 2555 break;
dc529a4f 2556 }
de151cf6 2557
007cc8ac 2558 reg->obj = NULL;
de151cf6 2559 obj_priv->fence_reg = I915_FENCE_REG_NONE;
007cc8ac 2560 list_del_init(&reg->lru_list);
de151cf6
JB		 2561}
2562
52dc7d32
CW		 2563/**
2564 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2565 * to the buffer to finish, and then resets the fence register.
2566 * @obj: tiled object holding a fence register.
2cf34d7b 2567 * @bool: whether the wait upon the fence is interruptible
52dc7d32
CW		 2568 *
2569 * Zeroes out the fence register itself and clears out the associated
2570 * data structures in dev_priv and obj_priv.
2571 */
2572int
2cf34d7b
CW		 2573i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2574 bool interruptible)
52dc7d32
CW		 2575{
2576 struct drm_device *dev = obj->dev;
53640e1d 2577 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2578 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
53640e1d 2579 struct drm_i915_fence_reg *reg;
52dc7d32
CW		 2580
2581 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2582 return 0;
2583
10ae9bd2
DV		 2584 /* If we've changed tiling, GTT-mappings of the object
2585 * need to re-fault to ensure that the correct fence register
2586 * setup is in place.
2587 */
2588 i915_gem_release_mmap(obj);
2589
52dc7d32
CW		 2590 /* On the i915, GPU access to tiled buffers is via a fence,
2591 * therefore we must wait for any outstanding access to complete
2592 * before clearing the fence.
2593 */
53640e1d
CW		 2594 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2595 if (reg->gpu) {
52dc7d32
CW		 2596 int ret;
2597
2cf34d7b 2598 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
0bc23aad
CW		 2599 if (ret)
2600 return ret;
2601
2cf34d7b 2602 ret = i915_gem_object_wait_rendering(obj, interruptible);
0bc23aad 2603 if (ret)
52dc7d32 2604 return ret;
53640e1d
CW		 2605
2606 reg->gpu = false;
52dc7d32
CW		 2607 }
2608
4a726612 2609 i915_gem_object_flush_gtt_write_domain(obj);
0bc23aad 2610 i915_gem_clear_fence_reg(obj);
52dc7d32
CW		 2611
2612 return 0;
2613}
2614
673a394b
EA		 2615/**
2616 * Finds free space in the GTT aperture and binds the object there.
2617 */
2618static int
2619i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2620{
2621 struct drm_device *dev = obj->dev;
2622 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2623 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 2624 struct drm_mm_node *free_space;
4bdadb97 2625 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
07f73f69 2626 int ret;
673a394b 2627
bb6baf76 2628 if (obj_priv->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2629 DRM_ERROR("Attempting to bind a purgeable object\n");
2630 return -EINVAL;
2631 }
2632
673a394b 2633 if (alignment == 0)
0f973f27 2634 alignment = i915_gem_get_gtt_alignment(obj);
8d7773a3 2635 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
673a394b
EA		 2636 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2637 return -EINVAL;
2638 }
2639
654fc607
CW		 2640 /* If the object is bigger than the entire aperture, reject it early
2641 * before evicting everything in a vain attempt to find space.
2642 */
73aa808f 2643 if (obj->size > dev_priv->mm.gtt_total) {
654fc607
CW		 2644 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2645 return -E2BIG;
2646 }
2647
673a394b
EA		 2648 search_free:
2649 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2650 obj->size, alignment, 0);
2651 if (free_space != NULL) {
2652 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2653 alignment);
db3307a9 2654 if (obj_priv->gtt_space != NULL)
673a394b 2655 obj_priv->gtt_offset = obj_priv->gtt_space->start;
673a394b
EA		 2656 }
2657 if (obj_priv->gtt_space == NULL) {
2658 /* If the gtt is empty and we're still having trouble
2659 * fitting our object in, we're out of memory.
2660 */
0108a3ed 2661 ret = i915_gem_evict_something(dev, obj->size, alignment);
9731129c 2662 if (ret)
673a394b 2663 return ret;
9731129c 2664
673a394b
EA		 2665 goto search_free;
2666 }
2667
4bdadb97 2668 ret = i915_gem_object_get_pages(obj, gfpmask);
673a394b
EA		 2669 if (ret) {
2670 drm_mm_put_block(obj_priv->gtt_space);
2671 obj_priv->gtt_space = NULL;
07f73f69
CW		 2672
2673 if (ret == -ENOMEM) {
2674 /* first try to clear up some space from the GTT */
0108a3ed
DV		 2675 ret = i915_gem_evict_something(dev, obj->size,
2676 alignment);
07f73f69 2677 if (ret) {
07f73f69 2678 /* now try to shrink everyone else */
4bdadb97
CW		 2679 if (gfpmask) {
2680 gfpmask = 0;
2681 goto search_free;
07f73f69
CW		 2682 }
2683
2684 return ret;
2685 }
2686
2687 goto search_free;
2688 }
2689
673a394b
EA		 2690 return ret;
2691 }
2692
673a394b
EA		 2693 /* Create an AGP memory structure pointing at our pages, and bind it
2694 * into the GTT.
2695 */
2696 obj_priv->agp_mem = drm_agp_bind_pages(dev,
856fa198 2697 obj_priv->pages,
07f73f69 2698 obj->size >> PAGE_SHIFT,
ba1eb1d8
KP		 2699 obj_priv->gtt_offset,
2700 obj_priv->agp_type);
673a394b 2701 if (obj_priv->agp_mem == NULL) {
856fa198 2702 i915_gem_object_put_pages(obj);
673a394b
EA		 2703 drm_mm_put_block(obj_priv->gtt_space);
2704 obj_priv->gtt_space = NULL;
07f73f69 2705
0108a3ed 2706 ret = i915_gem_evict_something(dev, obj->size, alignment);
9731129c 2707 if (ret)
07f73f69 2708 return ret;
07f73f69
CW		 2709
2710 goto search_free;
673a394b 2711 }
673a394b 2712
bf1a1092
CW		 2713 /* keep track of bounds object by adding it to the inactive list */
2714 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
73aa808f 2715 i915_gem_info_add_gtt(dev_priv, obj->size);
bf1a1092 2716
673a394b
EA		 2717 /* Assert that the object is not currently in any GPU domain. As it
2718 * wasn't in the GTT, there shouldn't be any way it could have been in
2719 * a GPU cache
2720 */
21d509e3
CW		 2721 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2722 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2723
1c5d22f7
CW		 2724 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2725
673a394b
EA		 2726 return 0;
2727}
2728
2729void
2730i915_gem_clflush_object(struct drm_gem_object *obj)
2731{
23010e43 2732 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2733
2734 /* If we don't have a page list set up, then we're not pinned
2735 * to GPU, and we can ignore the cache flush because it'll happen
2736 * again at bind time.
2737 */
856fa198 2738 if (obj_priv->pages == NULL)
673a394b
EA		 2739 return;
2740
1c5d22f7 2741 trace_i915_gem_object_clflush(obj);
cfa16a0d 2742
856fa198 2743 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
673a394b
EA		 2744}
2745
e47c68e9 2746/** Flushes any GPU write domain for the object if it's dirty. */
2dafb1e0 2747static int
ba3d8d74
DV		 2748i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2749 bool pipelined)
e47c68e9
EA		 2750{
2751 struct drm_device *dev = obj->dev;
1c5d22f7 2752 uint32_t old_write_domain;
e47c68e9
EA		 2753
2754 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2dafb1e0 2755 return 0;
e47c68e9
EA		 2756
2757 /* Queue the GPU write cache flushing we need. */
1c5d22f7 2758 old_write_domain = obj->write_domain;
c78ec30b 2759 i915_gem_flush_ring(dev, NULL,
9220434a
CW		 2760 to_intel_bo(obj)->ring,
2761 0, obj->write_domain);
48b956c5 2762 BUG_ON(obj->write_domain);
1c5d22f7
CW		 2763
2764 trace_i915_gem_object_change_domain(obj,
2765 obj->read_domains,
2766 old_write_domain);
ba3d8d74
DV		 2767
2768 if (pipelined)
2769 return 0;
2770
2cf34d7b 2771 return i915_gem_object_wait_rendering(obj, true);
e47c68e9
EA		 2772}
2773
2774/** Flushes the GTT write domain for the object if it's dirty. */
2775static void
2776i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2777{
1c5d22f7
CW		 2778 uint32_t old_write_domain;
2779
e47c68e9
EA		 2780 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2781 return;
2782
2783 /* No actual flushing is required for the GTT write domain. Writes
2784 * to it immediately go to main memory as far as we know, so there's
2785 * no chipset flush. It also doesn't land in render cache.
2786 */
1c5d22f7 2787 old_write_domain = obj->write_domain;
e47c68e9 2788 obj->write_domain = 0;
1c5d22f7
CW		 2789
2790 trace_i915_gem_object_change_domain(obj,
2791 obj->read_domains,
2792 old_write_domain);
e47c68e9
EA		 2793}
2794
2795/** Flushes the CPU write domain for the object if it's dirty. */
2796static void
2797i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2798{
2799 struct drm_device *dev = obj->dev;
1c5d22f7 2800 uint32_t old_write_domain;
e47c68e9
EA		 2801
2802 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2803 return;
2804
2805 i915_gem_clflush_object(obj);
2806 drm_agp_chipset_flush(dev);
1c5d22f7 2807 old_write_domain = obj->write_domain;
e47c68e9 2808 obj->write_domain = 0;
1c5d22f7
CW		 2809
2810 trace_i915_gem_object_change_domain(obj,
2811 obj->read_domains,
2812 old_write_domain);
e47c68e9
EA		 2813}
2814
2ef7eeaa
EA		 2815/**
2816 * Moves a single object to the GTT read, and possibly write domain.
2817 *
2818 * This function returns when the move is complete, including waiting on
2819 * flushes to occur.
2820 */
79e53945 2821int
2ef7eeaa
EA		 2822i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2823{
23010e43 2824 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 2825 uint32_t old_write_domain, old_read_domains;
e47c68e9 2826 int ret;
2ef7eeaa 2827
02354392
EA		 2828 /* Not valid to be called on unbound objects. */
2829 if (obj_priv->gtt_space == NULL)
2830 return -EINVAL;
2831
ba3d8d74 2832 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9
EA		 2833 if (ret != 0)
2834 return ret;
2835
7213342d 2836 i915_gem_object_flush_cpu_write_domain(obj);
1c5d22f7 2837
ba3d8d74 2838 if (write) {
2cf34d7b 2839 ret = i915_gem_object_wait_rendering(obj, true);
ba3d8d74
DV		 2840 if (ret)
2841 return ret;
ba3d8d74 2842 }
2ef7eeaa 2843
7213342d
CW		 2844 old_write_domain = obj->write_domain;
2845 old_read_domains = obj->read_domains;
2ef7eeaa 2846
e47c68e9
EA		 2847 /* It should now be out of any other write domains, and we can update
2848 * the domain values for our changes.
2849 */
2850 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2851 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2852 if (write) {
7213342d 2853 obj->read_domains = I915_GEM_DOMAIN_GTT;
e47c68e9
EA		 2854 obj->write_domain = I915_GEM_DOMAIN_GTT;
2855 obj_priv->dirty = 1;
2ef7eeaa
EA		 2856 }
2857
1c5d22f7
CW		 2858 trace_i915_gem_object_change_domain(obj,
2859 old_read_domains,
2860 old_write_domain);
2861
e47c68e9
EA		 2862 return 0;
2863}
2864
b9241ea3
ZW		 2865/*
2866 * Prepare buffer for display plane. Use uninterruptible for possible flush
2867 * wait, as in modesetting process we're not supposed to be interrupted.
2868 */
2869int
48b956c5
CW		 2870i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2871 bool pipelined)
b9241ea3 2872{
23010e43 2873 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ba3d8d74 2874 uint32_t old_read_domains;
b9241ea3
ZW		 2875 int ret;
2876
2877 /* Not valid to be called on unbound objects. */
2878 if (obj_priv->gtt_space == NULL)
2879 return -EINVAL;
2880
ced270fa 2881 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
48b956c5 2882 if (ret)
e35a41de 2883 return ret;
b9241ea3 2884
ced270fa
CW		 2885 /* Currently, we are always called from an non-interruptible context. */
2886 if (!pipelined) {
2887 ret = i915_gem_object_wait_rendering(obj, false);
2888 if (ret)
2889 return ret;
2890 }
2891
b118c1e3
CW		 2892 i915_gem_object_flush_cpu_write_domain(obj);
2893
b9241ea3 2894 old_read_domains = obj->read_domains;
c78ec30b 2895 obj->read_domains |= I915_GEM_DOMAIN_GTT;
b9241ea3
ZW		 2896
2897 trace_i915_gem_object_change_domain(obj,
2898 old_read_domains,
ba3d8d74 2899 obj->write_domain);
b9241ea3
ZW		 2900
2901 return 0;
2902}
2903
e47c68e9
EA		 2904/**
2905 * Moves a single object to the CPU read, and possibly write domain.
2906 *
2907 * This function returns when the move is complete, including waiting on
2908 * flushes to occur.
2909 */
2910static int
2911i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2912{
1c5d22f7 2913 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 2914 int ret;
2915
ba3d8d74 2916 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9
EA		 2917 if (ret != 0)
2918 return ret;
2ef7eeaa 2919
e47c68e9 2920 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 2921
e47c68e9
EA		 2922 /* If we have a partially-valid cache of the object in the CPU,
2923 * finish invalidating it and free the per-page flags.
2ef7eeaa 2924 */
e47c68e9 2925 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 2926
7213342d 2927 if (write) {
2cf34d7b 2928 ret = i915_gem_object_wait_rendering(obj, true);
7213342d
CW		 2929 if (ret)
2930 return ret;
2931 }
2932
1c5d22f7
CW		 2933 old_write_domain = obj->write_domain;
2934 old_read_domains = obj->read_domains;
2935
e47c68e9
EA		 2936 /* Flush the CPU cache if it's still invalid. */
2937 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 2938 i915_gem_clflush_object(obj);
2ef7eeaa 2939
e47c68e9 2940 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 2941 }
2942
2943 /* It should now be out of any other write domains, and we can update
2944 * the domain values for our changes.
2945 */
e47c68e9
EA		 2946 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2947
2948 /* If we're writing through the CPU, then the GPU read domains will
2949 * need to be invalidated at next use.
2950 */
2951 if (write) {
c78ec30b 2952 obj->read_domains = I915_GEM_DOMAIN_CPU;
e47c68e9
EA		 2953 obj->write_domain = I915_GEM_DOMAIN_CPU;
2954 }
2ef7eeaa 2955
1c5d22f7
CW		 2956 trace_i915_gem_object_change_domain(obj,
2957 old_read_domains,
2958 old_write_domain);
2959
2ef7eeaa
EA		 2960 return 0;
2961}
2962
673a394b
EA		 2963/*
2964 * Set the next domain for the specified object. This
2965 * may not actually perform the necessary flushing/invaliding though,
2966 * as that may want to be batched with other set_domain operations
2967 *
2968 * This is (we hope) the only really tricky part of gem. The goal
2969 * is fairly simple -- track which caches hold bits of the object
2970 * and make sure they remain coherent. A few concrete examples may
2971 * help to explain how it works. For shorthand, we use the notation
2972 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2973 * a pair of read and write domain masks.
2974 *
2975 * Case 1: the batch buffer
2976 *
2977 * 1. Allocated
2978 * 2. Written by CPU
2979 * 3. Mapped to GTT
2980 * 4. Read by GPU
2981 * 5. Unmapped from GTT
2982 * 6. Freed
2983 *
2984 * Let's take these a step at a time
2985 *
2986 * 1. Allocated
2987 * Pages allocated from the kernel may still have
2988 * cache contents, so we set them to (CPU, CPU) always.
2989 * 2. Written by CPU (using pwrite)
2990 * The pwrite function calls set_domain (CPU, CPU) and
2991 * this function does nothing (as nothing changes)
2992 * 3. Mapped by GTT
2993 * This function asserts that the object is not
2994 * currently in any GPU-based read or write domains
2995 * 4. Read by GPU
2996 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2997 * As write_domain is zero, this function adds in the
2998 * current read domains (CPU+COMMAND, 0).
2999 * flush_domains is set to CPU.
3000 * invalidate_domains is set to COMMAND
3001 * clflush is run to get data out of the CPU caches
3002 * then i915_dev_set_domain calls i915_gem_flush to
3003 * emit an MI_FLUSH and drm_agp_chipset_flush
3004 * 5. Unmapped from GTT
3005 * i915_gem_object_unbind calls set_domain (CPU, CPU)
3006 * flush_domains and invalidate_domains end up both zero
3007 * so no flushing/invalidating happens
3008 * 6. Freed
3009 * yay, done
3010 *
3011 * Case 2: The shared render buffer
3012 *
3013 * 1. Allocated
3014 * 2. Mapped to GTT
3015 * 3. Read/written by GPU
3016 * 4. set_domain to (CPU,CPU)
3017 * 5. Read/written by CPU
3018 * 6. Read/written by GPU
3019 *
3020 * 1. Allocated
3021 * Same as last example, (CPU, CPU)
3022 * 2. Mapped to GTT
3023 * Nothing changes (assertions find that it is not in the GPU)
3024 * 3. Read/written by GPU
3025 * execbuffer calls set_domain (RENDER, RENDER)
3026 * flush_domains gets CPU
3027 * invalidate_domains gets GPU
3028 * clflush (obj)
3029 * MI_FLUSH and drm_agp_chipset_flush
3030 * 4. set_domain (CPU, CPU)
3031 * flush_domains gets GPU
3032 * invalidate_domains gets CPU
3033 * wait_rendering (obj) to make sure all drawing is complete.
3034 * This will include an MI_FLUSH to get the data from GPU
3035 * to memory
3036 * clflush (obj) to invalidate the CPU cache
3037 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3038 * 5. Read/written by CPU
3039 * cache lines are loaded and dirtied
3040 * 6. Read written by GPU
3041 * Same as last GPU access
3042 *
3043 * Case 3: The constant buffer
3044 *
3045 * 1. Allocated
3046 * 2. Written by CPU
3047 * 3. Read by GPU
3048 * 4. Updated (written) by CPU again
3049 * 5. Read by GPU
3050 *
3051 * 1. Allocated
3052 * (CPU, CPU)
3053 * 2. Written by CPU
3054 * (CPU, CPU)
3055 * 3. Read by GPU
3056 * (CPU+RENDER, 0)
3057 * flush_domains = CPU
3058 * invalidate_domains = RENDER
3059 * clflush (obj)
3060 * MI_FLUSH
3061 * drm_agp_chipset_flush
3062 * 4. Updated (written) by CPU again
3063 * (CPU, CPU)
3064 * flush_domains = 0 (no previous write domain)
3065 * invalidate_domains = 0 (no new read domains)
3066 * 5. Read by GPU
3067 * (CPU+RENDER, 0)
3068 * flush_domains = CPU
3069 * invalidate_domains = RENDER
3070 * clflush (obj)
3071 * MI_FLUSH
3072 * drm_agp_chipset_flush
3073 */
c0d90829 3074static void
8b0e378a 3075i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
673a394b
EA		 3076{
3077 struct drm_device *dev = obj->dev;
9220434a 3078 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 3079 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 3080 uint32_t invalidate_domains = 0;
3081 uint32_t flush_domains = 0;
1c5d22f7 3082 uint32_t old_read_domains;
e47c68e9 3083
652c393a
JB		 3084 intel_mark_busy(dev, obj);
3085
673a394b
EA		 3086 /*
3087 * If the object isn't moving to a new write domain,
3088 * let the object stay in multiple read domains
3089 */
8b0e378a
EA		 3090 if (obj->pending_write_domain == 0)
3091 obj->pending_read_domains |= obj->read_domains;
673a394b
EA		 3092 else
3093 obj_priv->dirty = 1;
3094
3095 /*
3096 * Flush the current write domain if
3097 * the new read domains don't match. Invalidate
3098 * any read domains which differ from the old
3099 * write domain
3100 */
8b0e378a
EA		 3101 if (obj->write_domain &&
3102 obj->write_domain != obj->pending_read_domains) {
673a394b 3103 flush_domains |= obj->write_domain;
8b0e378a
EA		 3104 invalidate_domains |=
3105 obj->pending_read_domains & ~obj->write_domain;
673a394b
EA		 3106 }
3107 /*
3108 * Invalidate any read caches which may have
3109 * stale data. That is, any new read domains.
3110 */
8b0e378a 3111 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3d2a812a 3112 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
673a394b 3113 i915_gem_clflush_object(obj);
673a394b 3114
1c5d22f7
CW		 3115 old_read_domains = obj->read_domains;
3116
efbeed96
EA		 3117 /* The actual obj->write_domain will be updated with
3118 * pending_write_domain after we emit the accumulated flush for all
3119 * of our domain changes in execbuffers (which clears objects'
3120 * write_domains). So if we have a current write domain that we
3121 * aren't changing, set pending_write_domain to that.
3122 */
3123 if (flush_domains == 0 && obj->pending_write_domain == 0)
3124 obj->pending_write_domain = obj->write_domain;
8b0e378a 3125 obj->read_domains = obj->pending_read_domains;
673a394b
EA		 3126
3127 dev->invalidate_domains |= invalidate_domains;
3128 dev->flush_domains |= flush_domains;
9220434a
CW		 3129 if (obj_priv->ring)
3130 dev_priv->mm.flush_rings |= obj_priv->ring->id;
1c5d22f7
CW		 3131
3132 trace_i915_gem_object_change_domain(obj,
3133 old_read_domains,
3134 obj->write_domain);
673a394b
EA		 3135}
3136
3137/**
e47c68e9 3138 * Moves the object from a partially CPU read to a full one.
673a394b 3139 *
e47c68e9
EA		 3140 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3141 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 3142 */
e47c68e9
EA		 3143static void
3144i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
673a394b 3145{
23010e43 3146 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 3147
e47c68e9
EA		 3148 if (!obj_priv->page_cpu_valid)
3149 return;
3150
3151 /* If we're partially in the CPU read domain, finish moving it in.
3152 */
3153 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3154 int i;
3155
3156 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3157 if (obj_priv->page_cpu_valid[i])
3158 continue;
856fa198 3159 drm_clflush_pages(obj_priv->pages + i, 1);
e47c68e9 3160 }
e47c68e9
EA		 3161 }
3162
3163 /* Free the page_cpu_valid mappings which are now stale, whether
3164 * or not we've got I915_GEM_DOMAIN_CPU.
3165 */
9a298b2a 3166 kfree(obj_priv->page_cpu_valid);
e47c68e9
EA		 3167 obj_priv->page_cpu_valid = NULL;
3168}
3169
3170/**
3171 * Set the CPU read domain on a range of the object.
3172 *
3173 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3174 * not entirely valid. The page_cpu_valid member of the object flags which
3175 * pages have been flushed, and will be respected by
3176 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3177 * of the whole object.
3178 *
3179 * This function returns when the move is complete, including waiting on
3180 * flushes to occur.
3181 */
3182static int
3183i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3184 uint64_t offset, uint64_t size)
3185{
23010e43 3186 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 3187 uint32_t old_read_domains;
e47c68e9 3188 int i, ret;
673a394b 3189
e47c68e9
EA		 3190 if (offset == 0 && size == obj->size)
3191 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 3192
ba3d8d74 3193 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9 3194 if (ret != 0)
6a47baa6 3195 return ret;
e47c68e9
EA		 3196 i915_gem_object_flush_gtt_write_domain(obj);
3197
3198 /* If we're already fully in the CPU read domain, we're done. */
3199 if (obj_priv->page_cpu_valid == NULL &&
3200 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3201 return 0;
673a394b 3202
e47c68e9
EA		 3203 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3204 * newly adding I915_GEM_DOMAIN_CPU
3205 */
673a394b 3206 if (obj_priv->page_cpu_valid == NULL) {
9a298b2a
EA		 3207 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3208 GFP_KERNEL);
e47c68e9
EA		 3209 if (obj_priv->page_cpu_valid == NULL)
3210 return -ENOMEM;
3211 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3212 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
673a394b
EA		 3213
3214 /* Flush the cache on any pages that are still invalid from the CPU's
3215 * perspective.
3216 */
e47c68e9
EA		 3217 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3218 i++) {
673a394b
EA		 3219 if (obj_priv->page_cpu_valid[i])
3220 continue;
3221
856fa198 3222 drm_clflush_pages(obj_priv->pages + i, 1);
673a394b
EA		 3223
3224 obj_priv->page_cpu_valid[i] = 1;
3225 }
3226
e47c68e9
EA		 3227 /* It should now be out of any other write domains, and we can update
3228 * the domain values for our changes.
3229 */
3230 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3231
1c5d22f7 3232 old_read_domains = obj->read_domains;
e47c68e9
EA		 3233 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3234
1c5d22f7
CW		 3235 trace_i915_gem_object_change_domain(obj,
3236 old_read_domains,
3237 obj->write_domain);
3238
673a394b
EA		 3239 return 0;
3240}
3241
673a394b
EA		 3242/**
3243 * Pin an object to the GTT and evaluate the relocations landing in it.
3244 */
3245static int
3246i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3247 struct drm_file *file_priv,
2549d6c2 3248 struct drm_i915_gem_exec_object2 *entry)
673a394b
EA		 3249{
3250 struct drm_device *dev = obj->dev;
0839ccb8 3251 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 3252 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2549d6c2 3253 struct drm_i915_gem_relocation_entry __user *user_relocs;
673a394b 3254 int i, ret;
76446cac
JB		 3255 bool need_fence;
3256
3257 need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3258 obj_priv->tiling_mode != I915_TILING_NONE;
3259
3260 /* Check fence reg constraints and rebind if necessary */
808b24d6
CW		 3261 if (need_fence &&
3262 !i915_gem_object_fence_offset_ok(obj,
3263 obj_priv->tiling_mode)) {
3264 ret = i915_gem_object_unbind(obj);
3265 if (ret)
3266 return ret;
3267 }
673a394b
EA		 3268
3269 /* Choose the GTT offset for our buffer and put it there. */
3270 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3271 if (ret)
3272 return ret;
3273
76446cac
JB		 3274 /*
3275 * Pre-965 chips need a fence register set up in order to
3276 * properly handle blits to/from tiled surfaces.
3277 */
3278 if (need_fence) {
53640e1d 3279 ret = i915_gem_object_get_fence_reg(obj, true);
76446cac 3280 if (ret != 0) {
76446cac
JB		 3281 i915_gem_object_unpin(obj);
3282 return ret;
3283 }
53640e1d
CW		 3284
3285 dev_priv->fence_regs[obj_priv->fence_reg].gpu = true;
76446cac
JB		 3286 }
3287
673a394b
EA		 3288 entry->offset = obj_priv->gtt_offset;
3289
673a394b
EA		 3290 /* Apply the relocations, using the GTT aperture to avoid cache
3291 * flushing requirements.
3292 */
2549d6c2 3293 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
673a394b 3294 for (i = 0; i < entry->relocation_count; i++) {
2549d6c2 3295 struct drm_i915_gem_relocation_entry reloc;
673a394b
EA		 3296 struct drm_gem_object *target_obj;
3297 struct drm_i915_gem_object *target_obj_priv;
673a394b 3298
2549d6c2
CW		 3299 ret = __copy_from_user_inatomic(&reloc,
3300 user_relocs+i,
3301 sizeof(reloc));
3302 if (ret) {
3303 i915_gem_object_unpin(obj);
3304 return -EFAULT;
3305 }
3306
673a394b 3307 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
2549d6c2 3308 reloc.target_handle);
673a394b
EA		 3309 if (target_obj == NULL) {
3310 i915_gem_object_unpin(obj);
bf79cb91 3311 return -ENOENT;
673a394b 3312 }
23010e43 3313 target_obj_priv = to_intel_bo(target_obj);
673a394b 3314
8542a0bb
CW		 3315#if WATCH_RELOC
3316 DRM_INFO("%s: obj %p offset %08x target %d "
3317 "read %08x write %08x gtt %08x "
3318 "presumed %08x delta %08x\n",
3319 __func__,
3320 obj,
2549d6c2
CW		 3321 (int) reloc.offset,
3322 (int) reloc.target_handle,
3323 (int) reloc.read_domains,
3324 (int) reloc.write_domain,
8542a0bb 3325 (int) target_obj_priv->gtt_offset,
2549d6c2
CW		 3326 (int) reloc.presumed_offset,
3327 reloc.delta);
8542a0bb
CW		 3328#endif
3329
673a394b
EA		 3330 /* The target buffer should have appeared before us in the
3331 * exec_object list, so it should have a GTT space bound by now.
3332 */
3333 if (target_obj_priv->gtt_space == NULL) {
3334 DRM_ERROR("No GTT space found for object %d\n",
2549d6c2 3335 reloc.target_handle);
673a394b
EA		 3336 drm_gem_object_unreference(target_obj);
3337 i915_gem_object_unpin(obj);
3338 return -EINVAL;
3339 }
3340
8542a0bb 3341 /* Validate that the target is in a valid r/w GPU domain */
2549d6c2 3342 if (reloc.write_domain & (reloc.write_domain - 1)) {
16edd550
DV		 3343 DRM_ERROR("reloc with multiple write domains: "
3344 "obj %p target %d offset %d "
3345 "read %08x write %08x",
2549d6c2
CW		 3346 obj, reloc.target_handle,
3347 (int) reloc.offset,
3348 reloc.read_domains,
3349 reloc.write_domain);
929f49bf
JL		 3350 drm_gem_object_unreference(target_obj);
3351 i915_gem_object_unpin(obj);
16edd550
DV		 3352 return -EINVAL;
3353 }
2549d6c2
CW		 3354 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
3355 reloc.read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 3356 DRM_ERROR("reloc with read/write CPU domains: "
3357 "obj %p target %d offset %d "
3358 "read %08x write %08x",
2549d6c2
CW		 3359 obj, reloc.target_handle,
3360 (int) reloc.offset,
3361 reloc.read_domains,
3362 reloc.write_domain);
491152b8
CW		 3363 drm_gem_object_unreference(target_obj);
3364 i915_gem_object_unpin(obj);
e47c68e9
EA		 3365 return -EINVAL;
3366 }
2549d6c2
CW		 3367 if (reloc.write_domain && target_obj->pending_write_domain &&
3368 reloc.write_domain != target_obj->pending_write_domain) {
673a394b
EA		 3369 DRM_ERROR("Write domain conflict: "
3370 "obj %p target %d offset %d "
3371 "new %08x old %08x\n",
2549d6c2
CW		 3372 obj, reloc.target_handle,
3373 (int) reloc.offset,
3374 reloc.write_domain,
673a394b
EA		 3375 target_obj->pending_write_domain);
3376 drm_gem_object_unreference(target_obj);
3377 i915_gem_object_unpin(obj);
3378 return -EINVAL;
3379 }
3380
2549d6c2
CW		 3381 target_obj->pending_read_domains |= reloc.read_domains;
3382 target_obj->pending_write_domain |= reloc.write_domain;
673a394b
EA		 3383
3384 /* If the relocation already has the right value in it, no
3385 * more work needs to be done.
3386 */
2549d6c2 3387 if (target_obj_priv->gtt_offset == reloc.presumed_offset) {
673a394b
EA		 3388 drm_gem_object_unreference(target_obj);
3389 continue;
3390 }
3391
8542a0bb 3392 /* Check that the relocation address is valid... */
2549d6c2 3393 if (reloc.offset > obj->size - 4) {
8542a0bb
CW		 3394 DRM_ERROR("Relocation beyond object bounds: "
3395 "obj %p target %d offset %d size %d.\n",
2549d6c2
CW		 3396 obj, reloc.target_handle,
3397 (int) reloc.offset, (int) obj->size);
8542a0bb
CW		 3398 drm_gem_object_unreference(target_obj);
3399 i915_gem_object_unpin(obj);
3400 return -EINVAL;
3401 }
2549d6c2 3402 if (reloc.offset & 3) {
8542a0bb
CW		 3403 DRM_ERROR("Relocation not 4-byte aligned: "
3404 "obj %p target %d offset %d.\n",
2549d6c2
CW		 3405 obj, reloc.target_handle,
3406 (int) reloc.offset);
8542a0bb
CW		 3407 drm_gem_object_unreference(target_obj);
3408 i915_gem_object_unpin(obj);
3409 return -EINVAL;
3410 }
3411
3412 /* and points to somewhere within the target object. */
2549d6c2 3413 if (reloc.delta >= target_obj->size) {
8542a0bb
CW		 3414 DRM_ERROR("Relocation beyond target object bounds: "
3415 "obj %p target %d delta %d size %d.\n",
2549d6c2
CW		 3416 obj, reloc.target_handle,
3417 (int) reloc.delta, (int) target_obj->size);
8542a0bb
CW		 3418 drm_gem_object_unreference(target_obj);
3419 i915_gem_object_unpin(obj);
3420 return -EINVAL;
3421 }
3422
f0c43d9b
CW		 3423 reloc.delta += target_obj_priv->gtt_offset;
3424 if (obj->write_domain == I915_GEM_DOMAIN_CPU) {
3425 uint32_t page_offset = reloc.offset & ~PAGE_MASK;
3426 char *vaddr;
673a394b 3427
f0c43d9b
CW		 3428 vaddr = kmap_atomic(obj_priv->pages[reloc.offset >> PAGE_SHIFT], KM_USER0);
3429 *(uint32_t *)(vaddr + page_offset) = reloc.delta;
3430 kunmap_atomic(vaddr, KM_USER0);
3431 } else {
3432 uint32_t __iomem *reloc_entry;
3433 void __iomem *reloc_page;
3434 int ret;
3435
3436 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3437 if (ret) {
3438 drm_gem_object_unreference(target_obj);
3439 i915_gem_object_unpin(obj);
3440 return ret;
3441 }
3442
3443 /* Map the page containing the relocation we're going to perform. */
3444 reloc.offset += obj_priv->gtt_offset;
3445 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3446 reloc.offset & PAGE_MASK,
3447 KM_USER0);
3448 reloc_entry = (uint32_t __iomem *)
3449 (reloc_page + (reloc.offset & ~PAGE_MASK));
3450 iowrite32(reloc.delta, reloc_entry);
3451 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3452 }
673a394b 3453
673a394b
EA		 3454 drm_gem_object_unreference(target_obj);
3455 }
3456
673a394b
EA		 3457 return 0;
3458}
3459
673a394b
EA		 3460/* Throttle our rendering by waiting until the ring has completed our requests
3461 * emitted over 20 msec ago.
3462 *
b962442e
EA		 3463 * Note that if we were to use the current jiffies each time around the loop,
3464 * we wouldn't escape the function with any frames outstanding if the time to
3465 * render a frame was over 20ms.
3466 *
673a394b
EA		 3467 * This should get us reasonable parallelism between CPU and GPU but also
3468 * relatively low latency when blocking on a particular request to finish.
3469 */
3470static int
f787a5f5 3471i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
673a394b 3472{
f787a5f5
CW		 3473 struct drm_i915_private *dev_priv = dev->dev_private;
3474 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e 3475 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
f787a5f5
CW		 3476 struct drm_i915_gem_request *request;
3477 struct intel_ring_buffer *ring = NULL;
3478 u32 seqno = 0;
3479 int ret;
673a394b 3480
1c25595f 3481 spin_lock(&file_priv->mm.lock);
f787a5f5 3482 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
b962442e
EA		 3483 if (time_after_eq(request->emitted_jiffies, recent_enough))
3484 break;
3485
f787a5f5
CW		 3486 ring = request->ring;
3487 seqno = request->seqno;
b962442e 3488 }
1c25595f 3489 spin_unlock(&file_priv->mm.lock);
f787a5f5
CW		 3490
3491 if (seqno == 0)
3492 return 0;
3493
3494 ret = 0;
3495 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
3496 /* And wait for the seqno passing without holding any locks and
3497 * causing extra latency for others. This is safe as the irq
3498 * generation is designed to be run atomically and so is
3499 * lockless.
3500 */
3501 ring->user_irq_get(dev, ring);
3502 ret = wait_event_interruptible(ring->irq_queue,
3503 i915_seqno_passed(ring->get_seqno(dev, ring), seqno)
3504 || atomic_read(&dev_priv->mm.wedged));
3505 ring->user_irq_put(dev, ring);
3506
3507 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3508 ret = -EIO;
3509 }
3510
3511 if (ret == 0)
3512 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
b962442e 3513
673a394b
EA		 3514 return ret;
3515}
3516
40a5f0de 3517static int
2549d6c2
CW		 3518i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3519 uint64_t exec_offset)
40a5f0de 3520{
2549d6c2 3521 uint32_t exec_start, exec_len;
40a5f0de 3522
2549d6c2
CW		 3523 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3524 exec_len = (uint32_t) exec->batch_len;
40a5f0de 3525
2549d6c2
CW		 3526 if ((exec_start | exec_len) & 0x7)
3527 return -EINVAL;
40a5f0de 3528
2549d6c2
CW		 3529 if (!exec_start)
3530 return -EINVAL;
40a5f0de 3531
2bc43b5c 3532 return 0;
40a5f0de
EA		 3533}
3534
3535static int
2549d6c2
CW		 3536validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3537 int count)
40a5f0de 3538{
2549d6c2 3539 int i;
40a5f0de 3540
2549d6c2
CW		 3541 for (i = 0; i < count; i++) {
3542 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3543 size_t length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry);
2bc43b5c 3544
2549d6c2
CW		 3545 if (!access_ok(VERIFY_READ, ptr, length))
3546 return -EFAULT;
40a5f0de 3547
2549d6c2
CW		 3548 if (fault_in_pages_readable(ptr, length))
3549 return -EFAULT;
40a5f0de
EA		 3550 }
3551
83d60795
CW		 3552 return 0;
3553}
3554
8dc5d147 3555static int
76446cac
JB		 3556i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3557 struct drm_file *file_priv,
3558 struct drm_i915_gem_execbuffer2 *args,
3559 struct drm_i915_gem_exec_object2 *exec_list)
673a394b
EA		 3560{
3561 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3562 struct drm_gem_object **object_list = NULL;
3563 struct drm_gem_object *batch_obj;
b70d11da 3564 struct drm_i915_gem_object *obj_priv;
201361a5 3565 struct drm_clip_rect *cliprects = NULL;
8dc5d147 3566 struct drm_i915_gem_request *request = NULL;
2549d6c2 3567 int ret, i, pinned = 0;
673a394b 3568 uint64_t exec_offset;
6b95a207 3569 int pin_tries, flips;
673a394b 3570
852835f3
ZN		 3571 struct intel_ring_buffer *ring = NULL;
3572
30dbf0c0
CW		 3573 ret = i915_gem_check_is_wedged(dev);
3574 if (ret)
3575 return ret;
3576
2549d6c2
CW		 3577 ret = validate_exec_list(exec_list, args->buffer_count);
3578 if (ret)
3579 return ret;
3580
673a394b
EA		 3581#if WATCH_EXEC
3582 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3583 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3584#endif
d1b851fc
ZN		 3585 if (args->flags & I915_EXEC_BSD) {
3586 if (!HAS_BSD(dev)) {
3587 DRM_ERROR("execbuf with wrong flag\n");
3588 return -EINVAL;
3589 }
3590 ring = &dev_priv->bsd_ring;
3591 } else {
3592 ring = &dev_priv->render_ring;
3593 }
3594
4f481ed2
EA		 3595 if (args->buffer_count < 1) {
3596 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3597 return -EINVAL;
3598 }
c8e0f93a 3599 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
76446cac
JB		 3600 if (object_list == NULL) {
3601 DRM_ERROR("Failed to allocate object list for %d buffers\n",
673a394b
EA		 3602 args->buffer_count);
3603 ret = -ENOMEM;
3604 goto pre_mutex_err;
3605 }
673a394b 3606
201361a5 3607 if (args->num_cliprects != 0) {
9a298b2a
EA		 3608 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3609 GFP_KERNEL);
a40e8d31
OA		 3610 if (cliprects == NULL) {
3611 ret = -ENOMEM;
201361a5 3612 goto pre_mutex_err;
a40e8d31 3613 }
201361a5
EA		 3614
3615 ret = copy_from_user(cliprects,
3616 (struct drm_clip_rect __user *)
3617 (uintptr_t) args->cliprects_ptr,
3618 sizeof(*cliprects) * args->num_cliprects);
3619 if (ret != 0) {
3620 DRM_ERROR("copy %d cliprects failed: %d\n",
3621 args->num_cliprects, ret);
c877cdce 3622 ret = -EFAULT;
201361a5
EA		 3623 goto pre_mutex_err;
3624 }
3625 }
3626
8dc5d147
CW		 3627 request = kzalloc(sizeof(*request), GFP_KERNEL);
3628 if (request == NULL) {
3629 ret = -ENOMEM;
3630 goto pre_mutex_err;
3631 }
3632
76c1dec1
CW		 3633 ret = i915_mutex_lock_interruptible(dev);
3634 if (ret)
3635 goto pre_mutex_err;
673a394b 3636
673a394b 3637 if (dev_priv->mm.suspended) {
673a394b 3638 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3639 ret = -EBUSY;
3640 goto pre_mutex_err;
673a394b
EA		 3641 }
3642
ac94a962 3643 /* Look up object handles */
673a394b
EA		 3644 for (i = 0; i < args->buffer_count; i++) {
3645 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3646 exec_list[i].handle);
3647 if (object_list[i] == NULL) {
3648 DRM_ERROR("Invalid object handle %d at index %d\n",
3649 exec_list[i].handle, i);
0ce907f8
CW		 3650 /* prevent error path from reading uninitialized data */
3651 args->buffer_count = i + 1;
bf79cb91 3652 ret = -ENOENT;
673a394b
EA		 3653 goto err;
3654 }
b70d11da 3655
23010e43 3656 obj_priv = to_intel_bo(object_list[i]);
b70d11da
KH		 3657 if (obj_priv->in_execbuffer) {
3658 DRM_ERROR("Object %p appears more than once in object list\n",
3659 object_list[i]);
0ce907f8
CW		 3660 /* prevent error path from reading uninitialized data */
3661 args->buffer_count = i + 1;
bf79cb91 3662 ret = -EINVAL;
b70d11da
KH		 3663 goto err;
3664 }
3665 obj_priv->in_execbuffer = true;
ac94a962 3666 }
673a394b 3667
ac94a962
KP		 3668 /* Pin and relocate */
3669 for (pin_tries = 0; ; pin_tries++) {
3670 ret = 0;
40a5f0de 3671
ac94a962
KP		 3672 for (i = 0; i < args->buffer_count; i++) {
3673 object_list[i]->pending_read_domains = 0;
3674 object_list[i]->pending_write_domain = 0;
3675 ret = i915_gem_object_pin_and_relocate(object_list[i],
3676 file_priv,
2549d6c2 3677 &exec_list[i]);
ac94a962
KP		 3678 if (ret)
3679 break;
3680 pinned = i + 1;
3681 }
3682 /* success */
3683 if (ret == 0)
3684 break;
3685
3686 /* error other than GTT full, or we've already tried again */
2939e1f5 3687 if (ret != -ENOSPC || pin_tries >= 1) {
07f73f69
CW		 3688 if (ret != -ERESTARTSYS) {
3689 unsigned long long total_size = 0;
3d1cc470
CW		 3690 int num_fences = 0;
3691 for (i = 0; i < args->buffer_count; i++) {
43b27f40 3692 obj_priv = to_intel_bo(object_list[i]);
3d1cc470 3693
07f73f69 3694 total_size += object_list[i]->size;
3d1cc470
CW		 3695 num_fences +=
3696 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3697 obj_priv->tiling_mode != I915_TILING_NONE;
3698 }
3699 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
07f73f69 3700 pinned+1, args->buffer_count,
3d1cc470
CW		 3701 total_size, num_fences,
3702 ret);
73aa808f
CW		 3703 DRM_ERROR("%u objects [%u pinned, %u GTT], "
3704 "%zu object bytes [%zu pinned], "
3705 "%zu /%zu gtt bytes\n",
3706 dev_priv->mm.object_count,
3707 dev_priv->mm.pin_count,
3708 dev_priv->mm.gtt_count,
3709 dev_priv->mm.object_memory,
3710 dev_priv->mm.pin_memory,
3711 dev_priv->mm.gtt_memory,
3712 dev_priv->mm.gtt_total);
07f73f69 3713 }
673a394b
EA		 3714 goto err;
3715 }
ac94a962
KP		 3716
3717 /* unpin all of our buffers */
3718 for (i = 0; i < pinned; i++)
3719 i915_gem_object_unpin(object_list[i]);
b1177636 3720 pinned = 0;
ac94a962
KP		 3721
3722 /* evict everyone we can from the aperture */
3723 ret = i915_gem_evict_everything(dev);
07f73f69 3724 if (ret && ret != -ENOSPC)
ac94a962 3725 goto err;
673a394b
EA		 3726 }
3727
3728 /* Set the pending read domains for the batch buffer to COMMAND */
3729 batch_obj = object_list[args->buffer_count-1];
5f26a2c7
CW		 3730 if (batch_obj->pending_write_domain) {
3731 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3732 ret = -EINVAL;
3733 goto err;
3734 }
3735 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
673a394b 3736
83d60795
CW		 3737 /* Sanity check the batch buffer, prior to moving objects */
3738 exec_offset = exec_list[args->buffer_count - 1].offset;
3739 ret = i915_gem_check_execbuffer (args, exec_offset);
3740 if (ret != 0) {
3741 DRM_ERROR("execbuf with invalid offset/length\n");
3742 goto err;
3743 }
3744
646f0f6e
KP		 3745 /* Zero the global flush/invalidate flags. These
3746 * will be modified as new domains are computed
3747 * for each object
3748 */
3749 dev->invalidate_domains = 0;
3750 dev->flush_domains = 0;
9220434a 3751 dev_priv->mm.flush_rings = 0;
646f0f6e 3752
673a394b
EA		 3753 for (i = 0; i < args->buffer_count; i++) {
3754 struct drm_gem_object *obj = object_list[i];
673a394b 3755
646f0f6e 3756 /* Compute new gpu domains and update invalidate/flush */
8b0e378a 3757 i915_gem_object_set_to_gpu_domain(obj);
673a394b
EA		 3758 }
3759
646f0f6e
KP		 3760 if (dev->invalidate_domains | dev->flush_domains) {
3761#if WATCH_EXEC
3762 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3763 __func__,
3764 dev->invalidate_domains,
3765 dev->flush_domains);
3766#endif
c78ec30b 3767 i915_gem_flush(dev, file_priv,
646f0f6e 3768 dev->invalidate_domains,
9220434a
CW		 3769 dev->flush_domains,
3770 dev_priv->mm.flush_rings);
a6910434
DV		 3771 }
3772
efbeed96
EA		 3773 for (i = 0; i < args->buffer_count; i++) {
3774 struct drm_gem_object *obj = object_list[i];
23010e43 3775 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 3776 uint32_t old_write_domain = obj->write_domain;
efbeed96
EA		 3777
3778 obj->write_domain = obj->pending_write_domain;
99fcb766
DV		 3779 if (obj->write_domain)
3780 list_move_tail(&obj_priv->gpu_write_list,
3781 &dev_priv->mm.gpu_write_list);
99fcb766 3782
1c5d22f7
CW		 3783 trace_i915_gem_object_change_domain(obj,
3784 obj->read_domains,
3785 old_write_domain);
efbeed96
EA		 3786 }
3787
673a394b
EA		 3788#if WATCH_COHERENCY
3789 for (i = 0; i < args->buffer_count; i++) {
3790 i915_gem_object_check_coherency(object_list[i],
3791 exec_list[i].handle);
3792 }
3793#endif
3794
673a394b 3795#if WATCH_EXEC
6911a9b8 3796 i915_gem_dump_object(batch_obj,
673a394b
EA		 3797 args->batch_len,
3798 __func__,
3799 ~0);
3800#endif
3801
e59f2bac
CW		 3802 /* Check for any pending flips. As we only maintain a flip queue depth
3803 * of 1, we can simply insert a WAIT for the next display flip prior
3804 * to executing the batch and avoid stalling the CPU.
3805 */
3806 flips = 0;
3807 for (i = 0; i < args->buffer_count; i++) {
3808 if (object_list[i]->write_domain)
3809 flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3810 }
3811 if (flips) {
3812 int plane, flip_mask;
3813
3814 for (plane = 0; flips >> plane; plane++) {
3815 if (((flips >> plane) & 1) == 0)
3816 continue;
3817
3818 if (plane)
3819 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3820 else
3821 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3822
3823 intel_ring_begin(dev, ring, 2);
3824 intel_ring_emit(dev, ring,
3825 MI_WAIT_FOR_EVENT | flip_mask);
3826 intel_ring_emit(dev, ring, MI_NOOP);
3827 intel_ring_advance(dev, ring);
3828 }
3829 }
3830
673a394b 3831 /* Exec the batchbuffer */
852835f3 3832 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
e59f2bac 3833 cliprects, exec_offset);
673a394b
EA		 3834 if (ret) {
3835 DRM_ERROR("dispatch failed %d\n", ret);
3836 goto err;
3837 }
3838
3839 /*
3840 * Ensure that the commands in the batch buffer are
3841 * finished before the interrupt fires
3842 */
8a1a49f9 3843 i915_retire_commands(dev, ring);
673a394b 3844
617dbe27
DV		 3845 for (i = 0; i < args->buffer_count; i++) {
3846 struct drm_gem_object *obj = object_list[i];
3847 obj_priv = to_intel_bo(obj);
3848
3849 i915_gem_object_move_to_active(obj, ring);
617dbe27 3850 }
a56ba56c 3851
5c12a07e 3852 i915_add_request(dev, file_priv, request, ring);
8dc5d147 3853 request = NULL;
673a394b 3854
673a394b 3855err:
aad87dff
JL		 3856 for (i = 0; i < pinned; i++)
3857 i915_gem_object_unpin(object_list[i]);
3858
b70d11da
KH		 3859 for (i = 0; i < args->buffer_count; i++) {
3860 if (object_list[i]) {
23010e43 3861 obj_priv = to_intel_bo(object_list[i]);
b70d11da
KH		 3862 obj_priv->in_execbuffer = false;
3863 }
aad87dff 3864 drm_gem_object_unreference(object_list[i]);
b70d11da 3865 }
673a394b 3866
673a394b
EA		 3867 mutex_unlock(&dev->struct_mutex);
3868
93533c29 3869pre_mutex_err:
8e7d2b2c 3870 drm_free_large(object_list);
9a298b2a 3871 kfree(cliprects);
8dc5d147 3872 kfree(request);
673a394b
EA		 3873
3874 return ret;
3875}
3876
76446cac
JB		 3877/*
3878 * Legacy execbuffer just creates an exec2 list from the original exec object
3879 * list array and passes it to the real function.
3880 */
3881int
3882i915_gem_execbuffer(struct drm_device *dev, void *data,
3883 struct drm_file *file_priv)
3884{
3885 struct drm_i915_gem_execbuffer *args = data;
3886 struct drm_i915_gem_execbuffer2 exec2;
3887 struct drm_i915_gem_exec_object *exec_list = NULL;
3888 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3889 int ret, i;
3890
3891#if WATCH_EXEC
3892 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3893 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3894#endif
3895
3896 if (args->buffer_count < 1) {
3897 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3898 return -EINVAL;
3899 }
3900
3901 /* Copy in the exec list from userland */
3902 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3903 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3904 if (exec_list == NULL || exec2_list == NULL) {
3905 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3906 args->buffer_count);
3907 drm_free_large(exec_list);
3908 drm_free_large(exec2_list);
3909 return -ENOMEM;
3910 }
3911 ret = copy_from_user(exec_list,
3912 (struct drm_i915_relocation_entry __user *)
3913 (uintptr_t) args->buffers_ptr,
3914 sizeof(*exec_list) * args->buffer_count);
3915 if (ret != 0) {
3916 DRM_ERROR("copy %d exec entries failed %d\n",
3917 args->buffer_count, ret);
3918 drm_free_large(exec_list);
3919 drm_free_large(exec2_list);
3920 return -EFAULT;
3921 }
3922
3923 for (i = 0; i < args->buffer_count; i++) {
3924 exec2_list[i].handle = exec_list[i].handle;
3925 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3926 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3927 exec2_list[i].alignment = exec_list[i].alignment;
3928 exec2_list[i].offset = exec_list[i].offset;
a6c45cf0 3929 if (INTEL_INFO(dev)->gen < 4)
76446cac
JB		 3930 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3931 else
3932 exec2_list[i].flags = 0;
3933 }
3934
3935 exec2.buffers_ptr = args->buffers_ptr;
3936 exec2.buffer_count = args->buffer_count;
3937 exec2.batch_start_offset = args->batch_start_offset;
3938 exec2.batch_len = args->batch_len;
3939 exec2.DR1 = args->DR1;
3940 exec2.DR4 = args->DR4;
3941 exec2.num_cliprects = args->num_cliprects;
3942 exec2.cliprects_ptr = args->cliprects_ptr;
852835f3 3943 exec2.flags = I915_EXEC_RENDER;
76446cac
JB		 3944
3945 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3946 if (!ret) {
3947 /* Copy the new buffer offsets back to the user's exec list. */
3948 for (i = 0; i < args->buffer_count; i++)
3949 exec_list[i].offset = exec2_list[i].offset;
3950 /* ... and back out to userspace */
3951 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3952 (uintptr_t) args->buffers_ptr,
3953 exec_list,
3954 sizeof(*exec_list) * args->buffer_count);
3955 if (ret) {
3956 ret = -EFAULT;
3957 DRM_ERROR("failed to copy %d exec entries "
3958 "back to user (%d)\n",
3959 args->buffer_count, ret);
3960 }
76446cac
JB		 3961 }
3962
3963 drm_free_large(exec_list);
3964 drm_free_large(exec2_list);
3965 return ret;
3966}
3967
3968int
3969i915_gem_execbuffer2(struct drm_device *dev, void *data,
3970 struct drm_file *file_priv)
3971{
3972 struct drm_i915_gem_execbuffer2 *args = data;
3973 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3974 int ret;
3975
3976#if WATCH_EXEC
3977 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3978 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3979#endif
3980
3981 if (args->buffer_count < 1) {
3982 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
3983 return -EINVAL;
3984 }
3985
3986 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3987 if (exec2_list == NULL) {
3988 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3989 args->buffer_count);
3990 return -ENOMEM;
3991 }
3992 ret = copy_from_user(exec2_list,
3993 (struct drm_i915_relocation_entry __user *)
3994 (uintptr_t) args->buffers_ptr,
3995 sizeof(*exec2_list) * args->buffer_count);
3996 if (ret != 0) {
3997 DRM_ERROR("copy %d exec entries failed %d\n",
3998 args->buffer_count, ret);
3999 drm_free_large(exec2_list);
4000 return -EFAULT;
4001 }
4002
4003 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4004 if (!ret) {
4005 /* Copy the new buffer offsets back to the user's exec list. */
4006 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4007 (uintptr_t) args->buffers_ptr,
4008 exec2_list,
4009 sizeof(*exec2_list) * args->buffer_count);
4010 if (ret) {
4011 ret = -EFAULT;
4012 DRM_ERROR("failed to copy %d exec entries "
4013 "back to user (%d)\n",
4014 args->buffer_count, ret);
4015 }
4016 }
4017
4018 drm_free_large(exec2_list);
4019 return ret;
4020}
4021
673a394b
EA		 4022int
4023i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4024{
4025 struct drm_device *dev = obj->dev;
f13d3f73 4026 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 4027 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 4028 int ret;
4029
778c3544 4030 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
23bc5982 4031 WARN_ON(i915_verify_lists(dev));
ac0c6b5a
CW		 4032
4033 if (obj_priv->gtt_space != NULL) {
4034 if (alignment == 0)
4035 alignment = i915_gem_get_gtt_alignment(obj);
4036 if (obj_priv->gtt_offset & (alignment - 1)) {
ae7d49d8
CW		 4037 WARN(obj_priv->pin_count,
4038 "bo is already pinned with incorrect alignment:"
4039 " offset=%x, req.alignment=%x\n",
4040 obj_priv->gtt_offset, alignment);
ac0c6b5a
CW		 4041 ret = i915_gem_object_unbind(obj);
4042 if (ret)
4043 return ret;
4044 }
4045 }
4046
673a394b
EA		 4047 if (obj_priv->gtt_space == NULL) {
4048 ret = i915_gem_object_bind_to_gtt(obj, alignment);
9731129c 4049 if (ret)
673a394b 4050 return ret;
22c344e9 4051 }
76446cac 4052
673a394b
EA		 4053 obj_priv->pin_count++;
4054
4055 /* If the object is not active and not pending a flush,
4056 * remove it from the inactive list
4057 */
4058 if (obj_priv->pin_count == 1) {
73aa808f 4059 i915_gem_info_add_pin(dev_priv, obj->size);
f13d3f73
CW		 4060 if (!obj_priv->active)
4061 list_move_tail(&obj_priv->list,
4062 &dev_priv->mm.pinned_list);
673a394b 4063 }
673a394b 4064
23bc5982 4065 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 4066 return 0;
4067}
4068
4069void
4070i915_gem_object_unpin(struct drm_gem_object *obj)
4071{
4072 struct drm_device *dev = obj->dev;
4073 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 4074 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 4075
23bc5982 4076 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 4077 obj_priv->pin_count--;
4078 BUG_ON(obj_priv->pin_count < 0);
4079 BUG_ON(obj_priv->gtt_space == NULL);
4080
4081 /* If the object is no longer pinned, and is
4082 * neither active nor being flushed, then stick it on
4083 * the inactive list
4084 */
4085 if (obj_priv->pin_count == 0) {
f13d3f73 4086 if (!obj_priv->active)
673a394b
EA		 4087 list_move_tail(&obj_priv->list,
4088 &dev_priv->mm.inactive_list);
73aa808f 4089 i915_gem_info_remove_pin(dev_priv, obj->size);
673a394b 4090 }
23bc5982 4091 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 4092}
4093
4094int
4095i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4096 struct drm_file *file_priv)
4097{
4098 struct drm_i915_gem_pin *args = data;
4099 struct drm_gem_object *obj;
4100 struct drm_i915_gem_object *obj_priv;
4101 int ret;
4102
673a394b
EA		 4103 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4104 if (obj == NULL) {
4105 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4106 args->handle);
bf79cb91 4107 return -ENOENT;
673a394b 4108 }
23010e43 4109 obj_priv = to_intel_bo(obj);
673a394b 4110
76c1dec1
CW		 4111 ret = i915_mutex_lock_interruptible(dev);
4112 if (ret) {
4113 drm_gem_object_unreference_unlocked(obj);
4114 return ret;
4115 }
4116
bb6baf76
CW		 4117 if (obj_priv->madv != I915_MADV_WILLNEED) {
4118 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3ef94daa
CW		 4119 drm_gem_object_unreference(obj);
4120 mutex_unlock(&dev->struct_mutex);
4121 return -EINVAL;
4122 }
4123
79e53945
JB		 4124 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4125 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4126 args->handle);
96dec61d 4127 drm_gem_object_unreference(obj);
673a394b 4128 mutex_unlock(&dev->struct_mutex);
79e53945
JB		 4129 return -EINVAL;
4130 }
4131
4132 obj_priv->user_pin_count++;
4133 obj_priv->pin_filp = file_priv;
4134 if (obj_priv->user_pin_count == 1) {
4135 ret = i915_gem_object_pin(obj, args->alignment);
4136 if (ret != 0) {
4137 drm_gem_object_unreference(obj);
4138 mutex_unlock(&dev->struct_mutex);
4139 return ret;
4140 }
673a394b
EA		 4141 }
4142
4143 /* XXX - flush the CPU caches for pinned objects
4144 * as the X server doesn't manage domains yet
4145 */
e47c68e9 4146 i915_gem_object_flush_cpu_write_domain(obj);
673a394b
EA		 4147 args->offset = obj_priv->gtt_offset;
4148 drm_gem_object_unreference(obj);
4149 mutex_unlock(&dev->struct_mutex);
4150
4151 return 0;
4152}
4153
4154int
4155i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4156 struct drm_file *file_priv)
4157{
4158 struct drm_i915_gem_pin *args = data;
4159 struct drm_gem_object *obj;
79e53945 4160 struct drm_i915_gem_object *obj_priv;
76c1dec1 4161 int ret;
673a394b
EA		 4162
4163 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4164 if (obj == NULL) {
4165 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4166 args->handle);
bf79cb91 4167 return -ENOENT;
673a394b
EA		 4168 }
4169
23010e43 4170 obj_priv = to_intel_bo(obj);
76c1dec1
CW		 4171
4172 ret = i915_mutex_lock_interruptible(dev);
4173 if (ret) {
4174 drm_gem_object_unreference_unlocked(obj);
4175 return ret;
4176 }
4177
79e53945
JB		 4178 if (obj_priv->pin_filp != file_priv) {
4179 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4180 args->handle);
4181 drm_gem_object_unreference(obj);
4182 mutex_unlock(&dev->struct_mutex);
4183 return -EINVAL;
4184 }
4185 obj_priv->user_pin_count--;
4186 if (obj_priv->user_pin_count == 0) {
4187 obj_priv->pin_filp = NULL;
4188 i915_gem_object_unpin(obj);
4189 }
673a394b
EA		 4190
4191 drm_gem_object_unreference(obj);
4192 mutex_unlock(&dev->struct_mutex);
4193 return 0;
4194}
4195
4196int
4197i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4198 struct drm_file *file_priv)
4199{
4200 struct drm_i915_gem_busy *args = data;
4201 struct drm_gem_object *obj;
4202 struct drm_i915_gem_object *obj_priv;
30dbf0c0
CW		 4203 int ret;
4204
673a394b
EA		 4205 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4206 if (obj == NULL) {
4207 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4208 args->handle);
bf79cb91 4209 return -ENOENT;
673a394b
EA		 4210 }
4211
76c1dec1
CW		 4212 ret = i915_mutex_lock_interruptible(dev);
4213 if (ret) {
4214 drm_gem_object_unreference_unlocked(obj);
4215 return ret;
30dbf0c0
CW		 4216 }
4217
0be555b6
CW		 4218 /* Count all active objects as busy, even if they are currently not used
4219 * by the gpu. Users of this interface expect objects to eventually
4220 * become non-busy without any further actions, therefore emit any
4221 * necessary flushes here.
c4de0a5d 4222 */
0be555b6
CW		 4223 obj_priv = to_intel_bo(obj);
4224 args->busy = obj_priv->active;
4225 if (args->busy) {
4226 /* Unconditionally flush objects, even when the gpu still uses this
4227 * object. Userspace calling this function indicates that it wants to
4228 * use this buffer rather sooner than later, so issuing the required
4229 * flush earlier is beneficial.
4230 */
c78ec30b
CW		 4231 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4232 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 4233 obj_priv->ring,
4234 0, obj->write_domain);
0be555b6
CW		 4235
4236 /* Update the active list for the hardware's current position.
4237 * Otherwise this only updates on a delayed timer or when irqs
4238 * are actually unmasked, and our working set ends up being
4239 * larger than required.
4240 */
4241 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4242
4243 args->busy = obj_priv->active;
4244 }
673a394b
EA		 4245
4246 drm_gem_object_unreference(obj);
4247 mutex_unlock(&dev->struct_mutex);
76c1dec1 4248 return 0;
673a394b
EA		 4249}
4250
4251int
4252i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4253 struct drm_file *file_priv)
4254{
4255 return i915_gem_ring_throttle(dev, file_priv);
4256}
4257
3ef94daa
CW		 4258int
4259i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4260 struct drm_file *file_priv)
4261{
4262 struct drm_i915_gem_madvise *args = data;
4263 struct drm_gem_object *obj;
4264 struct drm_i915_gem_object *obj_priv;
76c1dec1 4265 int ret;
3ef94daa
CW		 4266
4267 switch (args->madv) {
4268 case I915_MADV_DONTNEED:
4269 case I915_MADV_WILLNEED:
4270 break;
4271 default:
4272 return -EINVAL;
4273 }
4274
4275 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4276 if (obj == NULL) {
4277 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4278 args->handle);
bf79cb91 4279 return -ENOENT;
3ef94daa 4280 }
23010e43 4281 obj_priv = to_intel_bo(obj);
3ef94daa 4282
76c1dec1
CW		 4283 ret = i915_mutex_lock_interruptible(dev);
4284 if (ret) {
4285 drm_gem_object_unreference_unlocked(obj);
4286 return ret;
4287 }
4288
3ef94daa
CW		 4289 if (obj_priv->pin_count) {
4290 drm_gem_object_unreference(obj);
4291 mutex_unlock(&dev->struct_mutex);
4292
4293 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4294 return -EINVAL;
4295 }
4296
bb6baf76
CW		 4297 if (obj_priv->madv != __I915_MADV_PURGED)
4298 obj_priv->madv = args->madv;
3ef94daa 4299
2d7ef395
CW		 4300 /* if the object is no longer bound, discard its backing storage */
4301 if (i915_gem_object_is_purgeable(obj_priv) &&
4302 obj_priv->gtt_space == NULL)
4303 i915_gem_object_truncate(obj);
4304
bb6baf76
CW		 4305 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4306
3ef94daa
CW		 4307 drm_gem_object_unreference(obj);
4308 mutex_unlock(&dev->struct_mutex);
4309
4310 return 0;
4311}
4312
ac52bc56
DV		 4313struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4314 size_t size)
4315{
73aa808f 4316 struct drm_i915_private *dev_priv = dev->dev_private;
c397b908 4317 struct drm_i915_gem_object *obj;
ac52bc56 4318
c397b908
DV		 4319 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4320 if (obj == NULL)
4321 return NULL;
673a394b 4322
c397b908
DV		 4323 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4324 kfree(obj);
4325 return NULL;
4326 }
673a394b 4327
73aa808f
CW		 4328 i915_gem_info_add_obj(dev_priv, size);
4329
c397b908
DV		 4330 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4331 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
673a394b 4332
c397b908 4333 obj->agp_type = AGP_USER_MEMORY;
62b8b215 4334 obj->base.driver_private = NULL;
c397b908
DV		 4335 obj->fence_reg = I915_FENCE_REG_NONE;
4336 INIT_LIST_HEAD(&obj->list);
4337 INIT_LIST_HEAD(&obj->gpu_write_list);
c397b908 4338 obj->madv = I915_MADV_WILLNEED;
de151cf6 4339
c397b908
DV		 4340 return &obj->base;
4341}
4342
4343int i915_gem_init_object(struct drm_gem_object *obj)
4344{
4345 BUG();
de151cf6 4346
673a394b
EA		 4347 return 0;
4348}
4349
be72615b 4350static void i915_gem_free_object_tail(struct drm_gem_object *obj)
673a394b 4351{
de151cf6 4352 struct drm_device *dev = obj->dev;
be72615b 4353 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 4354 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
be72615b 4355 int ret;
673a394b 4356
be72615b
CW		 4357 ret = i915_gem_object_unbind(obj);
4358 if (ret == -ERESTARTSYS) {
4359 list_move(&obj_priv->list,
4360 &dev_priv->mm.deferred_free_list);
4361 return;
4362 }
673a394b 4363
7e616158
CW		 4364 if (obj_priv->mmap_offset)
4365 i915_gem_free_mmap_offset(obj);
de151cf6 4366
c397b908 4367 drm_gem_object_release(obj);
73aa808f 4368 i915_gem_info_remove_obj(dev_priv, obj->size);
c397b908 4369
9a298b2a 4370 kfree(obj_priv->page_cpu_valid);
280b713b 4371 kfree(obj_priv->bit_17);
c397b908 4372 kfree(obj_priv);
673a394b
EA		 4373}
4374
be72615b
CW		 4375void i915_gem_free_object(struct drm_gem_object *obj)
4376{
4377 struct drm_device *dev = obj->dev;
4378 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4379
4380 trace_i915_gem_object_destroy(obj);
4381
4382 while (obj_priv->pin_count > 0)
4383 i915_gem_object_unpin(obj);
4384
4385 if (obj_priv->phys_obj)
4386 i915_gem_detach_phys_object(dev, obj);
4387
4388 i915_gem_free_object_tail(obj);
4389}
4390
29105ccc
CW		 4391int
4392i915_gem_idle(struct drm_device *dev)
4393{
4394 drm_i915_private_t *dev_priv = dev->dev_private;
4395 int ret;
28dfe52a 4396
29105ccc 4397 mutex_lock(&dev->struct_mutex);
1c5d22f7 4398
8187a2b7 4399 if (dev_priv->mm.suspended ||
d1b851fc
ZN		 4400 (dev_priv->render_ring.gem_object == NULL) ||
4401 (HAS_BSD(dev) &&
4402 dev_priv->bsd_ring.gem_object == NULL)) {
29105ccc
CW		 4403 mutex_unlock(&dev->struct_mutex);
4404 return 0;
28dfe52a
EA		 4405 }
4406
29105ccc 4407 ret = i915_gpu_idle(dev);
6dbe2772
KP		 4408 if (ret) {
4409 mutex_unlock(&dev->struct_mutex);
673a394b 4410 return ret;
6dbe2772 4411 }
673a394b 4412
29105ccc
CW		 4413 /* Under UMS, be paranoid and evict. */
4414 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
b47eb4a2 4415 ret = i915_gem_evict_inactive(dev);
29105ccc
CW		 4416 if (ret) {
4417 mutex_unlock(&dev->struct_mutex);
4418 return ret;
4419 }
4420 }
4421
4422 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4423 * We need to replace this with a semaphore, or something.
4424 * And not confound mm.suspended!
4425 */
4426 dev_priv->mm.suspended = 1;
bc0c7f14 4427 del_timer_sync(&dev_priv->hangcheck_timer);
29105ccc
CW		 4428
4429 i915_kernel_lost_context(dev);
6dbe2772 4430 i915_gem_cleanup_ringbuffer(dev);
29105ccc 4431
6dbe2772
KP		 4432 mutex_unlock(&dev->struct_mutex);
4433
29105ccc
CW		 4434 /* Cancel the retire work handler, which should be idle now. */
4435 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4436
673a394b
EA		 4437 return 0;
4438}
4439
e552eb70
JB		 4440/*
4441 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4442 * over cache flushing.
4443 */
8187a2b7 4444static int
e552eb70
JB		 4445i915_gem_init_pipe_control(struct drm_device *dev)
4446{
4447 drm_i915_private_t *dev_priv = dev->dev_private;
4448 struct drm_gem_object *obj;
4449 struct drm_i915_gem_object *obj_priv;
4450 int ret;
4451
34dc4d44 4452 obj = i915_gem_alloc_object(dev, 4096);
e552eb70
JB		 4453 if (obj == NULL) {
4454 DRM_ERROR("Failed to allocate seqno page\n");
4455 ret = -ENOMEM;
4456 goto err;
4457 }
4458 obj_priv = to_intel_bo(obj);
4459 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4460
4461 ret = i915_gem_object_pin(obj, 4096);
4462 if (ret)
4463 goto err_unref;
4464
4465 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4466 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4467 if (dev_priv->seqno_page == NULL)
4468 goto err_unpin;
4469
4470 dev_priv->seqno_obj = obj;
4471 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4472
4473 return 0;
4474
4475err_unpin:
4476 i915_gem_object_unpin(obj);
4477err_unref:
4478 drm_gem_object_unreference(obj);
4479err:
4480 return ret;
4481}
4482
8187a2b7
ZN		 4483
4484static void
e552eb70
JB		 4485i915_gem_cleanup_pipe_control(struct drm_device *dev)
4486{
4487 drm_i915_private_t *dev_priv = dev->dev_private;
4488 struct drm_gem_object *obj;
4489 struct drm_i915_gem_object *obj_priv;
4490
4491 obj = dev_priv->seqno_obj;
4492 obj_priv = to_intel_bo(obj);
4493 kunmap(obj_priv->pages[0]);
4494 i915_gem_object_unpin(obj);
4495 drm_gem_object_unreference(obj);
4496 dev_priv->seqno_obj = NULL;
4497
4498 dev_priv->seqno_page = NULL;
673a394b
EA		 4499}
4500
8187a2b7
ZN		 4501int
4502i915_gem_init_ringbuffer(struct drm_device *dev)
4503{
4504 drm_i915_private_t *dev_priv = dev->dev_private;
4505 int ret;
68f95ba9 4506
8187a2b7
ZN		 4507 if (HAS_PIPE_CONTROL(dev)) {
4508 ret = i915_gem_init_pipe_control(dev);
4509 if (ret)
4510 return ret;
4511 }
68f95ba9 4512
5c1143bb 4513 ret = intel_init_render_ring_buffer(dev);
68f95ba9
CW		 4514 if (ret)
4515 goto cleanup_pipe_control;
4516
4517 if (HAS_BSD(dev)) {
5c1143bb 4518 ret = intel_init_bsd_ring_buffer(dev);
68f95ba9
CW		 4519 if (ret)
4520 goto cleanup_render_ring;
d1b851fc 4521 }
68f95ba9 4522
6f392d54
CW		 4523 dev_priv->next_seqno = 1;
4524
68f95ba9
CW		 4525 return 0;
4526
4527cleanup_render_ring:
4528 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4529cleanup_pipe_control:
4530 if (HAS_PIPE_CONTROL(dev))
4531 i915_gem_cleanup_pipe_control(dev);
8187a2b7
ZN		 4532 return ret;
4533}
4534
4535void
4536i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4537{
4538 drm_i915_private_t *dev_priv = dev->dev_private;
4539
4540 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
d1b851fc
ZN		 4541 if (HAS_BSD(dev))
4542 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
8187a2b7
ZN		 4543 if (HAS_PIPE_CONTROL(dev))
4544 i915_gem_cleanup_pipe_control(dev);
4545}
4546
673a394b
EA		 4547int
4548i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4549 struct drm_file *file_priv)
4550{
4551 drm_i915_private_t *dev_priv = dev->dev_private;
4552 int ret;
4553
79e53945
JB		 4554 if (drm_core_check_feature(dev, DRIVER_MODESET))
4555 return 0;
4556
ba1234d1 4557 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 4558 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 4559 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 4560 }
4561
673a394b 4562 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 4563 dev_priv->mm.suspended = 0;
4564
4565 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 4566 if (ret != 0) {
4567 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 4568 return ret;
d816f6ac 4569 }
9bb2d6f9 4570
852835f3 4571 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
d1b851fc 4572 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
673a394b
EA		 4573 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4574 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
852835f3 4575 BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
d1b851fc 4576 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
673a394b 4577 mutex_unlock(&dev->struct_mutex);
dbb19d30 4578
5f35308b
CW		 4579 ret = drm_irq_install(dev);
4580 if (ret)
4581 goto cleanup_ringbuffer;
dbb19d30 4582
673a394b 4583 return 0;
5f35308b
CW		 4584
4585cleanup_ringbuffer:
4586 mutex_lock(&dev->struct_mutex);
4587 i915_gem_cleanup_ringbuffer(dev);
4588 dev_priv->mm.suspended = 1;
4589 mutex_unlock(&dev->struct_mutex);
4590
4591 return ret;
673a394b
EA		 4592}
4593
4594int
4595i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4596 struct drm_file *file_priv)
4597{
79e53945
JB		 4598 if (drm_core_check_feature(dev, DRIVER_MODESET))
4599 return 0;
4600
dbb19d30 4601 drm_irq_uninstall(dev);
e6890f6f 4602 return i915_gem_idle(dev);
673a394b
EA		 4603}
4604
4605void
4606i915_gem_lastclose(struct drm_device *dev)
4607{
4608 int ret;
673a394b 4609
e806b495
EA		 4610 if (drm_core_check_feature(dev, DRIVER_MODESET))
4611 return;
4612
6dbe2772
KP		 4613 ret = i915_gem_idle(dev);
4614 if (ret)
4615 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 4616}
4617
4618void
4619i915_gem_load(struct drm_device *dev)
4620{
b5aa8a0f 4621 int i;
673a394b
EA		 4622 drm_i915_private_t *dev_priv = dev->dev_private;
4623
673a394b 4624 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
99fcb766 4625 INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
673a394b 4626 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
f13d3f73 4627 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
a09ba7fa 4628 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
be72615b 4629 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
852835f3
ZN		 4630 INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4631 INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
d1b851fc
ZN		 4632 if (HAS_BSD(dev)) {
4633 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4634 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4635 }
007cc8ac
DV		 4636 for (i = 0; i < 16; i++)
4637 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
673a394b
EA		 4638 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4639 i915_gem_retire_work_handler);
30dbf0c0 4640 init_completion(&dev_priv->error_completion);
31169714
CW		 4641 spin_lock(&shrink_list_lock);
4642 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4643 spin_unlock(&shrink_list_lock);
4644
94400120
DA		 4645 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4646 if (IS_GEN3(dev)) {
4647 u32 tmp = I915_READ(MI_ARB_STATE);
4648 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4649 /* arb state is a masked write, so set bit + bit in mask */
4650 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4651 I915_WRITE(MI_ARB_STATE, tmp);
4652 }
4653 }
4654
de151cf6 4655 /* Old X drivers will take 0-2 for front, back, depth buffers */
b397c836
EA		 4656 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4657 dev_priv->fence_reg_start = 3;
de151cf6 4658
a6c45cf0 4659 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 4660 dev_priv->num_fence_regs = 16;
4661 else
4662 dev_priv->num_fence_regs = 8;
4663
b5aa8a0f 4664 /* Initialize fence registers to zero */
a6c45cf0
CW		 4665 switch (INTEL_INFO(dev)->gen) {
4666 case 6:
4667 for (i = 0; i < 16; i++)
4668 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4669 break;
4670 case 5:
4671 case 4:
b5aa8a0f
GH		 4672 for (i = 0; i < 16; i++)
4673 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
a6c45cf0
CW		 4674 break;
4675 case 3:
b5aa8a0f
GH		 4676 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4677 for (i = 0; i < 8; i++)
4678 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
a6c45cf0
CW		 4679 case 2:
4680 for (i = 0; i < 8; i++)
4681 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4682 break;
b5aa8a0f 4683 }
673a394b 4684 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 4685 init_waitqueue_head(&dev_priv->pending_flip_queue);
673a394b 4686}
71acb5eb
DA		 4687
4688/*
4689 * Create a physically contiguous memory object for this object
4690 * e.g. for cursor + overlay regs
4691 */
995b6762
CW		 4692static int i915_gem_init_phys_object(struct drm_device *dev,
4693 int id, int size, int align)
71acb5eb
DA		 4694{
4695 drm_i915_private_t *dev_priv = dev->dev_private;
4696 struct drm_i915_gem_phys_object *phys_obj;
4697 int ret;
4698
4699 if (dev_priv->mm.phys_objs[id - 1] || !size)
4700 return 0;
4701
9a298b2a 4702 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 4703 if (!phys_obj)
4704 return -ENOMEM;
4705
4706 phys_obj->id = id;
4707
6eeefaf3 4708 phys_obj->handle = drm_pci_alloc(dev, size, align);
71acb5eb
DA		 4709 if (!phys_obj->handle) {
4710 ret = -ENOMEM;
4711 goto kfree_obj;
4712 }
4713#ifdef CONFIG_X86
4714 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4715#endif
4716
4717 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4718
4719 return 0;
4720kfree_obj:
9a298b2a 4721 kfree(phys_obj);
71acb5eb
DA		 4722 return ret;
4723}
4724
995b6762 4725static void i915_gem_free_phys_object(struct drm_device *dev, int id)
71acb5eb
DA		 4726{
4727 drm_i915_private_t *dev_priv = dev->dev_private;
4728 struct drm_i915_gem_phys_object *phys_obj;
4729
4730 if (!dev_priv->mm.phys_objs[id - 1])
4731 return;
4732
4733 phys_obj = dev_priv->mm.phys_objs[id - 1];
4734 if (phys_obj->cur_obj) {
4735 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4736 }
4737
4738#ifdef CONFIG_X86
4739 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4740#endif
4741 drm_pci_free(dev, phys_obj->handle);
4742 kfree(phys_obj);
4743 dev_priv->mm.phys_objs[id - 1] = NULL;
4744}
4745
4746void i915_gem_free_all_phys_object(struct drm_device *dev)
4747{
4748 int i;
4749
260883c8 4750 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 4751 i915_gem_free_phys_object(dev, i);
4752}
4753
4754void i915_gem_detach_phys_object(struct drm_device *dev,
4755 struct drm_gem_object *obj)
4756{
4757 struct drm_i915_gem_object *obj_priv;
4758 int i;
4759 int ret;
4760 int page_count;
4761
23010e43 4762 obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4763 if (!obj_priv->phys_obj)
4764 return;
4765
4bdadb97 4766 ret = i915_gem_object_get_pages(obj, 0);
71acb5eb
DA		 4767 if (ret)
4768 goto out;
4769
4770 page_count = obj->size / PAGE_SIZE;
4771
4772 for (i = 0; i < page_count; i++) {
856fa198 4773 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4774 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4775
4776 memcpy(dst, src, PAGE_SIZE);
4777 kunmap_atomic(dst, KM_USER0);
4778 }
856fa198 4779 drm_clflush_pages(obj_priv->pages, page_count);
71acb5eb 4780 drm_agp_chipset_flush(dev);
d78b47b9
CW		 4781
4782 i915_gem_object_put_pages(obj);
71acb5eb
DA		 4783out:
4784 obj_priv->phys_obj->cur_obj = NULL;
4785 obj_priv->phys_obj = NULL;
4786}
4787
4788int
4789i915_gem_attach_phys_object(struct drm_device *dev,
6eeefaf3
CW		 4790 struct drm_gem_object *obj,
4791 int id,
4792 int align)
71acb5eb
DA		 4793{
4794 drm_i915_private_t *dev_priv = dev->dev_private;
4795 struct drm_i915_gem_object *obj_priv;
4796 int ret = 0;
4797 int page_count;
4798 int i;
4799
4800 if (id > I915_MAX_PHYS_OBJECT)
4801 return -EINVAL;
4802
23010e43 4803 obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4804
4805 if (obj_priv->phys_obj) {
4806 if (obj_priv->phys_obj->id == id)
4807 return 0;
4808 i915_gem_detach_phys_object(dev, obj);
4809 }
4810
71acb5eb
DA		 4811 /* create a new object */
4812 if (!dev_priv->mm.phys_objs[id - 1]) {
4813 ret = i915_gem_init_phys_object(dev, id,
6eeefaf3 4814 obj->size, align);
71acb5eb 4815 if (ret) {
aeb565df 4816 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
71acb5eb
DA		 4817 goto out;
4818 }
4819 }
4820
4821 /* bind to the object */
4822 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4823 obj_priv->phys_obj->cur_obj = obj;
4824
4bdadb97 4825 ret = i915_gem_object_get_pages(obj, 0);
71acb5eb
DA		 4826 if (ret) {
4827 DRM_ERROR("failed to get page list\n");
4828 goto out;
4829 }
4830
4831 page_count = obj->size / PAGE_SIZE;
4832
4833 for (i = 0; i < page_count; i++) {
856fa198 4834 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4835 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4836
4837 memcpy(dst, src, PAGE_SIZE);
4838 kunmap_atomic(src, KM_USER0);
4839 }
4840
d78b47b9
CW		 4841 i915_gem_object_put_pages(obj);
4842
71acb5eb
DA		 4843 return 0;
4844out:
4845 return ret;
4846}
4847
4848static int
4849i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4850 struct drm_i915_gem_pwrite *args,
4851 struct drm_file *file_priv)
4852{
23010e43 4853 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4854 void *obj_addr;
4855 int ret;
4856 char __user *user_data;
4857
4858 user_data = (char __user *) (uintptr_t) args->data_ptr;
4859 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4860
44d98a61 4861 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
71acb5eb
DA		 4862 ret = copy_from_user(obj_addr, user_data, args->size);
4863 if (ret)
4864 return -EFAULT;
4865
4866 drm_agp_chipset_flush(dev);
4867 return 0;
4868}
b962442e 4869
f787a5f5 4870void i915_gem_release(struct drm_device *dev, struct drm_file *file)
b962442e 4871{
f787a5f5 4872 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e
EA		 4873
4874 /* Clean up our request list when the client is going away, so that
4875 * later retire_requests won't dereference our soon-to-be-gone
4876 * file_priv.
4877 */
1c25595f 4878 spin_lock(&file_priv->mm.lock);
f787a5f5
CW		 4879 while (!list_empty(&file_priv->mm.request_list)) {
4880 struct drm_i915_gem_request *request;
4881
4882 request = list_first_entry(&file_priv->mm.request_list,
4883 struct drm_i915_gem_request,
4884 client_list);
4885 list_del(&request->client_list);
4886 request->file_priv = NULL;
4887 }
1c25595f 4888 spin_unlock(&file_priv->mm.lock);
b962442e 4889}
31169714 4890
1637ef41
CW		 4891static int
4892i915_gpu_is_active(struct drm_device *dev)
4893{
4894 drm_i915_private_t *dev_priv = dev->dev_private;
4895 int lists_empty;
4896
1637ef41 4897 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
852835f3 4898 list_empty(&dev_priv->render_ring.active_list);
d1b851fc
ZN		 4899 if (HAS_BSD(dev))
4900 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
1637ef41
CW		 4901
4902 return !lists_empty;
4903}
4904
31169714 4905static int
7f8275d0 4906i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
31169714
CW		 4907{
4908 drm_i915_private_t *dev_priv, *next_dev;
4909 struct drm_i915_gem_object *obj_priv, *next_obj;
4910 int cnt = 0;
4911 int would_deadlock = 1;
4912
4913 /* "fast-path" to count number of available objects */
4914 if (nr_to_scan == 0) {
4915 spin_lock(&shrink_list_lock);
4916 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4917 struct drm_device *dev = dev_priv->dev;
4918
4919 if (mutex_trylock(&dev->struct_mutex)) {
4920 list_for_each_entry(obj_priv,
4921 &dev_priv->mm.inactive_list,
4922 list)
4923 cnt++;
4924 mutex_unlock(&dev->struct_mutex);
4925 }
4926 }
4927 spin_unlock(&shrink_list_lock);
4928
4929 return (cnt / 100) * sysctl_vfs_cache_pressure;
4930 }
4931
4932 spin_lock(&shrink_list_lock);
4933
1637ef41 4934rescan:
31169714
CW		 4935 /* first scan for clean buffers */
4936 list_for_each_entry_safe(dev_priv, next_dev,
4937 &shrink_list, mm.shrink_list) {
4938 struct drm_device *dev = dev_priv->dev;
4939
4940 if (! mutex_trylock(&dev->struct_mutex))
4941 continue;
4942
4943 spin_unlock(&shrink_list_lock);
b09a1fec 4944 i915_gem_retire_requests(dev);
31169714
CW		 4945
4946 list_for_each_entry_safe(obj_priv, next_obj,
4947 &dev_priv->mm.inactive_list,
4948 list) {
4949 if (i915_gem_object_is_purgeable(obj_priv)) {
a8089e84 4950 i915_gem_object_unbind(&obj_priv->base);
31169714
CW		 4951 if (--nr_to_scan <= 0)
4952 break;
4953 }
4954 }
4955
4956 spin_lock(&shrink_list_lock);
4957 mutex_unlock(&dev->struct_mutex);
4958
963b4836
CW		 4959 would_deadlock = 0;
4960
31169714
CW		 4961 if (nr_to_scan <= 0)
4962 break;
4963 }
4964
4965 /* second pass, evict/count anything still on the inactive list */
4966 list_for_each_entry_safe(dev_priv, next_dev,
4967 &shrink_list, mm.shrink_list) {
4968 struct drm_device *dev = dev_priv->dev;
4969
4970 if (! mutex_trylock(&dev->struct_mutex))
4971 continue;
4972
4973 spin_unlock(&shrink_list_lock);
4974
4975 list_for_each_entry_safe(obj_priv, next_obj,
4976 &dev_priv->mm.inactive_list,
4977 list) {
4978 if (nr_to_scan > 0) {
a8089e84 4979 i915_gem_object_unbind(&obj_priv->base);
31169714
CW		 4980 nr_to_scan--;
4981 } else
4982 cnt++;
4983 }
4984
4985 spin_lock(&shrink_list_lock);
4986 mutex_unlock(&dev->struct_mutex);
4987
4988 would_deadlock = 0;
4989 }
4990
1637ef41
CW		 4991 if (nr_to_scan) {
4992 int active = 0;
4993
4994 /*
4995 * We are desperate for pages, so as a last resort, wait
4996 * for the GPU to finish and discard whatever we can.
4997 * This has a dramatic impact to reduce the number of
4998 * OOM-killer events whilst running the GPU aggressively.
4999 */
5000 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5001 struct drm_device *dev = dev_priv->dev;
5002
5003 if (!mutex_trylock(&dev->struct_mutex))
5004 continue;
5005
5006 spin_unlock(&shrink_list_lock);
5007
5008 if (i915_gpu_is_active(dev)) {
5009 i915_gpu_idle(dev);
5010 active++;
5011 }
5012
5013 spin_lock(&shrink_list_lock);
5014 mutex_unlock(&dev->struct_mutex);
5015 }
5016
5017 if (active)
5018 goto rescan;
5019 }
5020
31169714
CW		 5021 spin_unlock(&shrink_list_lock);
5022
5023 if (would_deadlock)
5024 return -1;
5025 else if (cnt > 0)
5026 return (cnt / 100) * sysctl_vfs_cache_pressure;
5027 else
5028 return 0;
5029}
5030
5031static struct shrinker shrinker = {
5032 .shrink = i915_gem_shrink,
5033 .seeks = DEFAULT_SEEKS,
5034};
5035
5036__init void
5037i915_gem_shrinker_init(void)
5038{
5039 register_shrinker(&shrinker);
5040}
5041
5042__exit void
5043i915_gem_shrinker_exit(void)
5044{
5045 unregister_shrinker(&shrinker);
5046}
Linux kernel - Deliverables
This page took 0.583574 seconds and 5 git commands to generate.