projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: kill ring->setup_status_page
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
5a0e3ad6 34#include <linux/slab.h>
673a394b 35#include <linux/swap.h>
79e53945 36#include <linux/pci.h>
f8f235e5 37#include <linux/intel-gtt.h>
673a394b 38
0108a3ed 39static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
ba3d8d74
DV		 40
41static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
42 bool pipelined);
e47c68e9
EA		 43static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
44static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
e47c68e9
EA		 45static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
46 int write);
47static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
48 uint64_t offset,
49 uint64_t size);
50static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
2cf34d7b
CW		 51static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
52 bool interruptible);
de151cf6
JB		 53static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
54 unsigned alignment);
de151cf6 55static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
71acb5eb
DA		 56static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
57 struct drm_i915_gem_pwrite *args,
58 struct drm_file *file_priv);
be72615b 59static void i915_gem_free_object_tail(struct drm_gem_object *obj);
673a394b 60
31169714
CW		 61static LIST_HEAD(shrink_list);
62static DEFINE_SPINLOCK(shrink_list_lock);
63
30dbf0c0
CW		 64int
65i915_gem_check_is_wedged(struct drm_device *dev)
66{
67 struct drm_i915_private *dev_priv = dev->dev_private;
68 struct completion *x = &dev_priv->error_completion;
69 unsigned long flags;
70 int ret;
71
72 if (!atomic_read(&dev_priv->mm.wedged))
73 return 0;
74
75 ret = wait_for_completion_interruptible(x);
76 if (ret)
77 return ret;
78
79 /* Success, we reset the GPU! */
80 if (!atomic_read(&dev_priv->mm.wedged))
81 return 0;
82
83 /* GPU is hung, bump the completion count to account for
84 * the token we just consumed so that we never hit zero and
85 * end up waiting upon a subsequent completion event that
86 * will never happen.
87 */
88 spin_lock_irqsave(&x->wait.lock, flags);
89 x->done++;
90 spin_unlock_irqrestore(&x->wait.lock, flags);
91 return -EIO;
92}
93
76c1dec1
CW		 94static int i915_mutex_lock_interruptible(struct drm_device *dev)
95{
96 struct drm_i915_private *dev_priv = dev->dev_private;
97 int ret;
98
99 ret = i915_gem_check_is_wedged(dev);
100 if (ret)
101 return ret;
102
103 ret = mutex_lock_interruptible(&dev->struct_mutex);
104 if (ret)
105 return ret;
106
107 if (atomic_read(&dev_priv->mm.wedged)) {
108 mutex_unlock(&dev->struct_mutex);
109 return -EAGAIN;
110 }
111
112 return 0;
113}
30dbf0c0 114
7d1c4804
CW		 115static inline bool
116i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
117{
118 return obj_priv->gtt_space &&
119 !obj_priv->active &&
120 obj_priv->pin_count == 0;
121}
122
79e53945
JB		 123int i915_gem_do_init(struct drm_device *dev, unsigned long start,
124 unsigned long end)
673a394b
EA		 125{
126 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 127
79e53945
JB		 128 if (start >= end ||
129 (start & (PAGE_SIZE - 1)) != 0 ||
130 (end & (PAGE_SIZE - 1)) != 0) {
673a394b
EA		 131 return -EINVAL;
132 }
133
79e53945
JB		 134 drm_mm_init(&dev_priv->mm.gtt_space, start,
135 end - start);
673a394b 136
79e53945
JB		 137 dev->gtt_total = (uint32_t) (end - start);
138
139 return 0;
140}
673a394b 141
79e53945
JB		 142int
143i915_gem_init_ioctl(struct drm_device *dev, void *data,
144 struct drm_file *file_priv)
145{
146 struct drm_i915_gem_init *args = data;
147 int ret;
148
149 mutex_lock(&dev->struct_mutex);
150 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
673a394b
EA		 151 mutex_unlock(&dev->struct_mutex);
152
79e53945 153 return ret;
673a394b
EA		 154}
155
5a125c3c
EA		 156int
157i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
158 struct drm_file *file_priv)
159{
5a125c3c 160 struct drm_i915_gem_get_aperture *args = data;
5a125c3c
EA		 161
162 if (!(dev->driver->driver_features & DRIVER_GEM))
163 return -ENODEV;
164
165 args->aper_size = dev->gtt_total;
2678d9d6
KP		 166 args->aper_available_size = (args->aper_size -
167 atomic_read(&dev->pin_memory));
5a125c3c
EA		 168
169 return 0;
170}
171
673a394b
EA		 172
173/**
174 * Creates a new mm object and returns a handle to it.
175 */
176int
177i915_gem_create_ioctl(struct drm_device *dev, void *data,
178 struct drm_file *file_priv)
179{
180 struct drm_i915_gem_create *args = data;
181 struct drm_gem_object *obj;
a1a2d1d3
PP		 182 int ret;
183 u32 handle;
673a394b
EA		 184
185 args->size = roundup(args->size, PAGE_SIZE);
186
187 /* Allocate the new object */
ac52bc56 188 obj = i915_gem_alloc_object(dev, args->size);
673a394b
EA		 189 if (obj == NULL)
190 return -ENOMEM;
191
192 ret = drm_gem_handle_create(file_priv, obj, &handle);
1dfd9754
CW		 193 if (ret) {
194 drm_gem_object_unreference_unlocked(obj);
673a394b 195 return ret;
1dfd9754 196 }
673a394b 197
1dfd9754
CW		 198 /* Sink the floating reference from kref_init(handlecount) */
199 drm_gem_object_handle_unreference_unlocked(obj);
673a394b 200
1dfd9754 201 args->handle = handle;
673a394b
EA		 202 return 0;
203}
204
eb01459f
EA		 205static inline int
206fast_shmem_read(struct page **pages,
207 loff_t page_base, int page_offset,
208 char __user *data,
209 int length)
210{
211 char __iomem *vaddr;
2bc43b5c 212 int unwritten;
eb01459f
EA		 213
214 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
215 if (vaddr == NULL)
216 return -ENOMEM;
2bc43b5c 217 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
eb01459f
EA		 218 kunmap_atomic(vaddr, KM_USER0);
219
2bc43b5c
FM		 220 if (unwritten)
221 return -EFAULT;
222
223 return 0;
eb01459f
EA		 224}
225
280b713b
EA		 226static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
227{
228 drm_i915_private_t *dev_priv = obj->dev->dev_private;
23010e43 229 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
280b713b
EA		 230
231 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
232 obj_priv->tiling_mode != I915_TILING_NONE;
233}
234
99a03df5 235static inline void
40123c1f
EA		 236slow_shmem_copy(struct page *dst_page,
237 int dst_offset,
238 struct page *src_page,
239 int src_offset,
240 int length)
241{
242 char *dst_vaddr, *src_vaddr;
243
99a03df5
CW		 244 dst_vaddr = kmap(dst_page);
245 src_vaddr = kmap(src_page);
40123c1f
EA		 246
247 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
248
99a03df5
CW		 249 kunmap(src_page);
250 kunmap(dst_page);
40123c1f
EA		 251}
252
99a03df5 253static inline void
280b713b
EA		 254slow_shmem_bit17_copy(struct page *gpu_page,
255 int gpu_offset,
256 struct page *cpu_page,
257 int cpu_offset,
258 int length,
259 int is_read)
260{
261 char *gpu_vaddr, *cpu_vaddr;
262
263 /* Use the unswizzled path if this page isn't affected. */
264 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
265 if (is_read)
266 return slow_shmem_copy(cpu_page, cpu_offset,
267 gpu_page, gpu_offset, length);
268 else
269 return slow_shmem_copy(gpu_page, gpu_offset,
270 cpu_page, cpu_offset, length);
271 }
272
99a03df5
CW		 273 gpu_vaddr = kmap(gpu_page);
274 cpu_vaddr = kmap(cpu_page);
280b713b
EA		 275
276 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
277 * XORing with the other bits (A9 for Y, A9 and A10 for X)
278 */
279 while (length > 0) {
280 int cacheline_end = ALIGN(gpu_offset + 1, 64);
281 int this_length = min(cacheline_end - gpu_offset, length);
282 int swizzled_gpu_offset = gpu_offset ^ 64;
283
284 if (is_read) {
285 memcpy(cpu_vaddr + cpu_offset,
286 gpu_vaddr + swizzled_gpu_offset,
287 this_length);
288 } else {
289 memcpy(gpu_vaddr + swizzled_gpu_offset,
290 cpu_vaddr + cpu_offset,
291 this_length);
292 }
293 cpu_offset += this_length;
294 gpu_offset += this_length;
295 length -= this_length;
296 }
297
99a03df5
CW		 298 kunmap(cpu_page);
299 kunmap(gpu_page);
280b713b
EA		 300}
301
eb01459f
EA		 302/**
303 * This is the fast shmem pread path, which attempts to copy_from_user directly
304 * from the backing pages of the object to the user's address space. On a
305 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
306 */
307static int
308i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
309 struct drm_i915_gem_pread *args,
310 struct drm_file *file_priv)
311{
23010e43 312 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
eb01459f
EA		 313 ssize_t remain;
314 loff_t offset, page_base;
315 char __user *user_data;
316 int page_offset, page_length;
317 int ret;
318
319 user_data = (char __user *) (uintptr_t) args->data_ptr;
320 remain = args->size;
321
76c1dec1
CW		 322 ret = i915_mutex_lock_interruptible(dev);
323 if (ret)
324 return ret;
eb01459f 325
4bdadb97 326 ret = i915_gem_object_get_pages(obj, 0);
eb01459f
EA		 327 if (ret != 0)
328 goto fail_unlock;
329
330 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
331 args->size);
332 if (ret != 0)
333 goto fail_put_pages;
334
23010e43 335 obj_priv = to_intel_bo(obj);
eb01459f
EA		 336 offset = args->offset;
337
338 while (remain > 0) {
339 /* Operation in this page
340 *
341 * page_base = page offset within aperture
342 * page_offset = offset within page
343 * page_length = bytes to copy for this page
344 */
345 page_base = (offset & ~(PAGE_SIZE-1));
346 page_offset = offset & (PAGE_SIZE-1);
347 page_length = remain;
348 if ((page_offset + remain) > PAGE_SIZE)
349 page_length = PAGE_SIZE - page_offset;
350
351 ret = fast_shmem_read(obj_priv->pages,
352 page_base, page_offset,
353 user_data, page_length);
354 if (ret)
355 goto fail_put_pages;
356
357 remain -= page_length;
358 user_data += page_length;
359 offset += page_length;
360 }
361
362fail_put_pages:
363 i915_gem_object_put_pages(obj);
364fail_unlock:
365 mutex_unlock(&dev->struct_mutex);
366
367 return ret;
368}
369
07f73f69
CW		 370static int
371i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
372{
373 int ret;
374
4bdadb97 375 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
07f73f69
CW		 376
377 /* If we've insufficient memory to map in the pages, attempt
378 * to make some space by throwing out some old buffers.
379 */
380 if (ret == -ENOMEM) {
381 struct drm_device *dev = obj->dev;
07f73f69 382
0108a3ed
DV		 383 ret = i915_gem_evict_something(dev, obj->size,
384 i915_gem_get_gtt_alignment(obj));
07f73f69
CW		 385 if (ret)
386 return ret;
387
4bdadb97 388 ret = i915_gem_object_get_pages(obj, 0);
07f73f69
CW		 389 }
390
391 return ret;
392}
393
eb01459f
EA		 394/**
395 * This is the fallback shmem pread path, which allocates temporary storage
396 * in kernel space to copy_to_user into outside of the struct_mutex, so we
397 * can copy out of the object's backing pages while holding the struct mutex
398 * and not take page faults.
399 */
400static int
401i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
402 struct drm_i915_gem_pread *args,
403 struct drm_file *file_priv)
404{
23010e43 405 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
eb01459f
EA		 406 struct mm_struct *mm = current->mm;
407 struct page **user_pages;
408 ssize_t remain;
409 loff_t offset, pinned_pages, i;
410 loff_t first_data_page, last_data_page, num_pages;
411 int shmem_page_index, shmem_page_offset;
412 int data_page_index, data_page_offset;
413 int page_length;
414 int ret;
415 uint64_t data_ptr = args->data_ptr;
280b713b 416 int do_bit17_swizzling;
eb01459f
EA		 417
418 remain = args->size;
419
420 /* Pin the user pages containing the data. We can't fault while
421 * holding the struct mutex, yet we want to hold it while
422 * dereferencing the user data.
423 */
424 first_data_page = data_ptr / PAGE_SIZE;
425 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
426 num_pages = last_data_page - first_data_page + 1;
427
8e7d2b2c 428 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
eb01459f
EA		 429 if (user_pages == NULL)
430 return -ENOMEM;
431
432 down_read(&mm->mmap_sem);
433 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 434 num_pages, 1, 0, user_pages, NULL);
eb01459f
EA		 435 up_read(&mm->mmap_sem);
436 if (pinned_pages < num_pages) {
437 ret = -EFAULT;
438 goto fail_put_user_pages;
439 }
440
280b713b
EA		 441 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
442
76c1dec1
CW		 443 ret = i915_mutex_lock_interruptible(dev);
444 if (ret)
445 goto fail_put_user_pages;
eb01459f 446
07f73f69
CW		 447 ret = i915_gem_object_get_pages_or_evict(obj);
448 if (ret)
eb01459f
EA		 449 goto fail_unlock;
450
451 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
452 args->size);
453 if (ret != 0)
454 goto fail_put_pages;
455
23010e43 456 obj_priv = to_intel_bo(obj);
eb01459f
EA		 457 offset = args->offset;
458
459 while (remain > 0) {
460 /* Operation in this page
461 *
462 * shmem_page_index = page number within shmem file
463 * shmem_page_offset = offset within page in shmem file
464 * data_page_index = page number in get_user_pages return
465 * data_page_offset = offset with data_page_index page.
466 * page_length = bytes to copy for this page
467 */
468 shmem_page_index = offset / PAGE_SIZE;
469 shmem_page_offset = offset & ~PAGE_MASK;
470 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
471 data_page_offset = data_ptr & ~PAGE_MASK;
472
473 page_length = remain;
474 if ((shmem_page_offset + page_length) > PAGE_SIZE)
475 page_length = PAGE_SIZE - shmem_page_offset;
476 if ((data_page_offset + page_length) > PAGE_SIZE)
477 page_length = PAGE_SIZE - data_page_offset;
478
280b713b 479 if (do_bit17_swizzling) {
99a03df5 480 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
280b713b 481 shmem_page_offset,
99a03df5
CW		 482 user_pages[data_page_index],
483 data_page_offset,
484 page_length,
485 1);
486 } else {
487 slow_shmem_copy(user_pages[data_page_index],
488 data_page_offset,
489 obj_priv->pages[shmem_page_index],
490 shmem_page_offset,
491 page_length);
280b713b 492 }
eb01459f
EA		 493
494 remain -= page_length;
495 data_ptr += page_length;
496 offset += page_length;
497 }
498
499fail_put_pages:
500 i915_gem_object_put_pages(obj);
501fail_unlock:
502 mutex_unlock(&dev->struct_mutex);
503fail_put_user_pages:
504 for (i = 0; i < pinned_pages; i++) {
505 SetPageDirty(user_pages[i]);
506 page_cache_release(user_pages[i]);
507 }
8e7d2b2c 508 drm_free_large(user_pages);
eb01459f
EA		 509
510 return ret;
511}
512
673a394b
EA		 513/**
514 * Reads data from the object referenced by handle.
515 *
516 * On error, the contents of *data are undefined.
517 */
518int
519i915_gem_pread_ioctl(struct drm_device *dev, void *data,
520 struct drm_file *file_priv)
521{
522 struct drm_i915_gem_pread *args = data;
523 struct drm_gem_object *obj;
524 struct drm_i915_gem_object *obj_priv;
673a394b
EA		 525 int ret;
526
527 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
528 if (obj == NULL)
bf79cb91 529 return -ENOENT;
23010e43 530 obj_priv = to_intel_bo(obj);
673a394b
EA		 531
532 /* Bounds check source.
533 *
534 * XXX: This could use review for overflow issues...
535 */
536 if (args->offset > obj->size || args->size > obj->size ||
537 args->offset + args->size > obj->size) {
bc9025bd 538 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 539 return -EINVAL;
540 }
541
280b713b 542 if (i915_gem_object_needs_bit17_swizzle(obj)) {
eb01459f 543 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
280b713b
EA		 544 } else {
545 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
546 if (ret != 0)
547 ret = i915_gem_shmem_pread_slow(dev, obj, args,
548 file_priv);
549 }
673a394b 550
bc9025bd 551 drm_gem_object_unreference_unlocked(obj);
673a394b 552
eb01459f 553 return ret;
673a394b
EA		 554}
555
0839ccb8
KP		 556/* This is the fast write path which cannot handle
557 * page faults in the source data
9b7530cc 558 */
0839ccb8
KP		 559
560static inline int
561fast_user_write(struct io_mapping *mapping,
562 loff_t page_base, int page_offset,
563 char __user *user_data,
564 int length)
9b7530cc 565{
9b7530cc 566 char *vaddr_atomic;
0839ccb8 567 unsigned long unwritten;
9b7530cc 568
fca3ec01 569 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
0839ccb8
KP		 570 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
571 user_data, length);
fca3ec01 572 io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
0839ccb8
KP		 573 if (unwritten)
574 return -EFAULT;
575 return 0;
576}
577
578/* Here's the write path which can sleep for
579 * page faults
580 */
581
ab34c226 582static inline void
3de09aa3
EA		 583slow_kernel_write(struct io_mapping *mapping,
584 loff_t gtt_base, int gtt_offset,
585 struct page *user_page, int user_offset,
586 int length)
0839ccb8 587{
ab34c226
CW		 588 char __iomem *dst_vaddr;
589 char *src_vaddr;
0839ccb8 590
ab34c226
CW		 591 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
592 src_vaddr = kmap(user_page);
593
594 memcpy_toio(dst_vaddr + gtt_offset,
595 src_vaddr + user_offset,
596 length);
597
598 kunmap(user_page);
599 io_mapping_unmap(dst_vaddr);
9b7530cc
LT		 600}
601
40123c1f
EA		 602static inline int
603fast_shmem_write(struct page **pages,
604 loff_t page_base, int page_offset,
605 char __user *data,
606 int length)
607{
608 char __iomem *vaddr;
d0088775 609 unsigned long unwritten;
40123c1f
EA		 610
611 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
612 if (vaddr == NULL)
613 return -ENOMEM;
d0088775 614 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
40123c1f
EA		 615 kunmap_atomic(vaddr, KM_USER0);
616
d0088775
DA		 617 if (unwritten)
618 return -EFAULT;
40123c1f
EA		 619 return 0;
620}
621
3de09aa3
EA		 622/**
623 * This is the fast pwrite path, where we copy the data directly from the
624 * user into the GTT, uncached.
625 */
673a394b 626static int
3de09aa3
EA		 627i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
628 struct drm_i915_gem_pwrite *args,
629 struct drm_file *file_priv)
673a394b 630{
23010e43 631 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
0839ccb8 632 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 633 ssize_t remain;
0839ccb8 634 loff_t offset, page_base;
673a394b 635 char __user *user_data;
0839ccb8
KP		 636 int page_offset, page_length;
637 int ret;
673a394b
EA		 638
639 user_data = (char __user *) (uintptr_t) args->data_ptr;
640 remain = args->size;
641 if (!access_ok(VERIFY_READ, user_data, remain))
642 return -EFAULT;
643
76c1dec1
CW		 644 ret = i915_mutex_lock_interruptible(dev);
645 if (ret)
646 return ret;
673a394b 647
673a394b
EA		 648 ret = i915_gem_object_pin(obj, 0);
649 if (ret) {
650 mutex_unlock(&dev->struct_mutex);
651 return ret;
652 }
2ef7eeaa 653 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
673a394b
EA		 654 if (ret)
655 goto fail;
656
23010e43 657 obj_priv = to_intel_bo(obj);
673a394b 658 offset = obj_priv->gtt_offset + args->offset;
673a394b
EA		 659
660 while (remain > 0) {
661 /* Operation in this page
662 *
0839ccb8
KP		 663 * page_base = page offset within aperture
664 * page_offset = offset within page
665 * page_length = bytes to copy for this page
673a394b 666 */
0839ccb8
KP		 667 page_base = (offset & ~(PAGE_SIZE-1));
668 page_offset = offset & (PAGE_SIZE-1);
669 page_length = remain;
670 if ((page_offset + remain) > PAGE_SIZE)
671 page_length = PAGE_SIZE - page_offset;
672
673 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
674 page_offset, user_data, page_length);
675
676 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 677 * source page isn't available. Return the error and we'll
678 * retry in the slow path.
0839ccb8 679 */
3de09aa3
EA		 680 if (ret)
681 goto fail;
673a394b 682
0839ccb8
KP		 683 remain -= page_length;
684 user_data += page_length;
685 offset += page_length;
673a394b 686 }
673a394b
EA		 687
688fail:
689 i915_gem_object_unpin(obj);
690 mutex_unlock(&dev->struct_mutex);
691
692 return ret;
693}
694
3de09aa3
EA		 695/**
696 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
697 * the memory and maps it using kmap_atomic for copying.
698 *
699 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
700 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
701 */
3043c60c 702static int
3de09aa3
EA		 703i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
704 struct drm_i915_gem_pwrite *args,
705 struct drm_file *file_priv)
673a394b 706{
23010e43 707 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3de09aa3
EA		 708 drm_i915_private_t *dev_priv = dev->dev_private;
709 ssize_t remain;
710 loff_t gtt_page_base, offset;
711 loff_t first_data_page, last_data_page, num_pages;
712 loff_t pinned_pages, i;
713 struct page **user_pages;
714 struct mm_struct *mm = current->mm;
715 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 716 int ret;
3de09aa3
EA		 717 uint64_t data_ptr = args->data_ptr;
718
719 remain = args->size;
720
721 /* Pin the user pages containing the data. We can't fault while
722 * holding the struct mutex, and all of the pwrite implementations
723 * want to hold it while dereferencing the user data.
724 */
725 first_data_page = data_ptr / PAGE_SIZE;
726 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
727 num_pages = last_data_page - first_data_page + 1;
728
8e7d2b2c 729 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
3de09aa3
EA		 730 if (user_pages == NULL)
731 return -ENOMEM;
732
733 down_read(&mm->mmap_sem);
734 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
735 num_pages, 0, 0, user_pages, NULL);
736 up_read(&mm->mmap_sem);
737 if (pinned_pages < num_pages) {
738 ret = -EFAULT;
739 goto out_unpin_pages;
740 }
673a394b 741
76c1dec1
CW		 742 ret = i915_mutex_lock_interruptible(dev);
743 if (ret)
744 goto out_unpin_pages;
745
3de09aa3
EA		 746 ret = i915_gem_object_pin(obj, 0);
747 if (ret)
748 goto out_unlock;
749
750 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
751 if (ret)
752 goto out_unpin_object;
753
23010e43 754 obj_priv = to_intel_bo(obj);
3de09aa3
EA		 755 offset = obj_priv->gtt_offset + args->offset;
756
757 while (remain > 0) {
758 /* Operation in this page
759 *
760 * gtt_page_base = page offset within aperture
761 * gtt_page_offset = offset within page in aperture
762 * data_page_index = page number in get_user_pages return
763 * data_page_offset = offset with data_page_index page.
764 * page_length = bytes to copy for this page
765 */
766 gtt_page_base = offset & PAGE_MASK;
767 gtt_page_offset = offset & ~PAGE_MASK;
768 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
769 data_page_offset = data_ptr & ~PAGE_MASK;
770
771 page_length = remain;
772 if ((gtt_page_offset + page_length) > PAGE_SIZE)
773 page_length = PAGE_SIZE - gtt_page_offset;
774 if ((data_page_offset + page_length) > PAGE_SIZE)
775 page_length = PAGE_SIZE - data_page_offset;
776
ab34c226
CW		 777 slow_kernel_write(dev_priv->mm.gtt_mapping,
778 gtt_page_base, gtt_page_offset,
779 user_pages[data_page_index],
780 data_page_offset,
781 page_length);
3de09aa3
EA		 782
783 remain -= page_length;
784 offset += page_length;
785 data_ptr += page_length;
786 }
787
788out_unpin_object:
789 i915_gem_object_unpin(obj);
790out_unlock:
791 mutex_unlock(&dev->struct_mutex);
792out_unpin_pages:
793 for (i = 0; i < pinned_pages; i++)
794 page_cache_release(user_pages[i]);
8e7d2b2c 795 drm_free_large(user_pages);
3de09aa3
EA		 796
797 return ret;
798}
799
40123c1f
EA		 800/**
801 * This is the fast shmem pwrite path, which attempts to directly
802 * copy_from_user into the kmapped pages backing the object.
803 */
3043c60c 804static int
40123c1f
EA		 805i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
806 struct drm_i915_gem_pwrite *args,
807 struct drm_file *file_priv)
673a394b 808{
23010e43 809 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
40123c1f
EA		 810 ssize_t remain;
811 loff_t offset, page_base;
812 char __user *user_data;
813 int page_offset, page_length;
673a394b 814 int ret;
40123c1f
EA		 815
816 user_data = (char __user *) (uintptr_t) args->data_ptr;
817 remain = args->size;
673a394b 818
76c1dec1
CW		 819 ret = i915_mutex_lock_interruptible(dev);
820 if (ret)
821 return ret;
673a394b 822
4bdadb97 823 ret = i915_gem_object_get_pages(obj, 0);
40123c1f
EA		 824 if (ret != 0)
825 goto fail_unlock;
673a394b 826
e47c68e9 827 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
40123c1f
EA		 828 if (ret != 0)
829 goto fail_put_pages;
830
23010e43 831 obj_priv = to_intel_bo(obj);
40123c1f
EA		 832 offset = args->offset;
833 obj_priv->dirty = 1;
834
835 while (remain > 0) {
836 /* Operation in this page
837 *
838 * page_base = page offset within aperture
839 * page_offset = offset within page
840 * page_length = bytes to copy for this page
841 */
842 page_base = (offset & ~(PAGE_SIZE-1));
843 page_offset = offset & (PAGE_SIZE-1);
844 page_length = remain;
845 if ((page_offset + remain) > PAGE_SIZE)
846 page_length = PAGE_SIZE - page_offset;
847
848 ret = fast_shmem_write(obj_priv->pages,
849 page_base, page_offset,
850 user_data, page_length);
851 if (ret)
852 goto fail_put_pages;
853
854 remain -= page_length;
855 user_data += page_length;
856 offset += page_length;
857 }
858
859fail_put_pages:
860 i915_gem_object_put_pages(obj);
861fail_unlock:
862 mutex_unlock(&dev->struct_mutex);
863
864 return ret;
865}
866
867/**
868 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
869 * the memory and maps it using kmap_atomic for copying.
870 *
871 * This avoids taking mmap_sem for faulting on the user's address while the
872 * struct_mutex is held.
873 */
874static int
875i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
876 struct drm_i915_gem_pwrite *args,
877 struct drm_file *file_priv)
878{
23010e43 879 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
40123c1f
EA		 880 struct mm_struct *mm = current->mm;
881 struct page **user_pages;
882 ssize_t remain;
883 loff_t offset, pinned_pages, i;
884 loff_t first_data_page, last_data_page, num_pages;
885 int shmem_page_index, shmem_page_offset;
886 int data_page_index, data_page_offset;
887 int page_length;
888 int ret;
889 uint64_t data_ptr = args->data_ptr;
280b713b 890 int do_bit17_swizzling;
40123c1f
EA		 891
892 remain = args->size;
893
894 /* Pin the user pages containing the data. We can't fault while
895 * holding the struct mutex, and all of the pwrite implementations
896 * want to hold it while dereferencing the user data.
897 */
898 first_data_page = data_ptr / PAGE_SIZE;
899 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
900 num_pages = last_data_page - first_data_page + 1;
901
8e7d2b2c 902 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
40123c1f
EA		 903 if (user_pages == NULL)
904 return -ENOMEM;
905
906 down_read(&mm->mmap_sem);
907 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
908 num_pages, 0, 0, user_pages, NULL);
909 up_read(&mm->mmap_sem);
910 if (pinned_pages < num_pages) {
911 ret = -EFAULT;
912 goto fail_put_user_pages;
673a394b
EA		 913 }
914
280b713b
EA		 915 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
916
76c1dec1
CW		 917 ret = i915_mutex_lock_interruptible(dev);
918 if (ret)
919 goto fail_put_user_pages;
40123c1f 920
07f73f69
CW		 921 ret = i915_gem_object_get_pages_or_evict(obj);
922 if (ret)
40123c1f
EA		 923 goto fail_unlock;
924
925 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
926 if (ret != 0)
927 goto fail_put_pages;
928
23010e43 929 obj_priv = to_intel_bo(obj);
673a394b 930 offset = args->offset;
40123c1f 931 obj_priv->dirty = 1;
673a394b 932
40123c1f
EA		 933 while (remain > 0) {
934 /* Operation in this page
935 *
936 * shmem_page_index = page number within shmem file
937 * shmem_page_offset = offset within page in shmem file
938 * data_page_index = page number in get_user_pages return
939 * data_page_offset = offset with data_page_index page.
940 * page_length = bytes to copy for this page
941 */
942 shmem_page_index = offset / PAGE_SIZE;
943 shmem_page_offset = offset & ~PAGE_MASK;
944 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
945 data_page_offset = data_ptr & ~PAGE_MASK;
946
947 page_length = remain;
948 if ((shmem_page_offset + page_length) > PAGE_SIZE)
949 page_length = PAGE_SIZE - shmem_page_offset;
950 if ((data_page_offset + page_length) > PAGE_SIZE)
951 page_length = PAGE_SIZE - data_page_offset;
952
280b713b 953 if (do_bit17_swizzling) {
99a03df5 954 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
280b713b
EA		 955 shmem_page_offset,
956 user_pages[data_page_index],
957 data_page_offset,
99a03df5
CW		 958 page_length,
959 0);
960 } else {
961 slow_shmem_copy(obj_priv->pages[shmem_page_index],
962 shmem_page_offset,
963 user_pages[data_page_index],
964 data_page_offset,
965 page_length);
280b713b 966 }
40123c1f
EA		 967
968 remain -= page_length;
969 data_ptr += page_length;
970 offset += page_length;
673a394b
EA		 971 }
972
40123c1f
EA		 973fail_put_pages:
974 i915_gem_object_put_pages(obj);
975fail_unlock:
673a394b 976 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 977fail_put_user_pages:
978 for (i = 0; i < pinned_pages; i++)
979 page_cache_release(user_pages[i]);
8e7d2b2c 980 drm_free_large(user_pages);
673a394b 981
40123c1f 982 return ret;
673a394b
EA		 983}
984
985/**
986 * Writes data to the object referenced by handle.
987 *
988 * On error, the contents of the buffer that were to be modified are undefined.
989 */
990int
991i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
992 struct drm_file *file_priv)
993{
994 struct drm_i915_gem_pwrite *args = data;
995 struct drm_gem_object *obj;
996 struct drm_i915_gem_object *obj_priv;
997 int ret = 0;
998
999 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1000 if (obj == NULL)
bf79cb91 1001 return -ENOENT;
23010e43 1002 obj_priv = to_intel_bo(obj);
673a394b
EA		 1003
1004 /* Bounds check destination.
1005 *
1006 * XXX: This could use review for overflow issues...
1007 */
1008 if (args->offset > obj->size || args->size > obj->size ||
1009 args->offset + args->size > obj->size) {
bc9025bd 1010 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1011 return -EINVAL;
1012 }
1013
1014 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1015 * it would end up going through the fenced access, and we'll get
1016 * different detiling behavior between reading and writing.
1017 * pread/pwrite currently are reading and writing from the CPU
1018 * perspective, requiring manual detiling by the client.
1019 */
71acb5eb
DA		 1020 if (obj_priv->phys_obj)
1021 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
1022 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
9b8c4a0b
CW		 1023 dev->gtt_total != 0 &&
1024 obj->write_domain != I915_GEM_DOMAIN_CPU) {
3de09aa3
EA		 1025 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
1026 if (ret == -EFAULT) {
1027 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
1028 file_priv);
1029 }
280b713b
EA		 1030 } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
1031 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
40123c1f
EA		 1032 } else {
1033 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
1034 if (ret == -EFAULT) {
1035 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
1036 file_priv);
1037 }
1038 }
673a394b
EA		 1039
1040#if WATCH_PWRITE
1041 if (ret)
1042 DRM_INFO("pwrite failed %d\n", ret);
1043#endif
1044
bc9025bd 1045 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1046
1047 return ret;
1048}
1049
1050/**
2ef7eeaa
EA		 1051 * Called when user space prepares to use an object with the CPU, either
1052 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 1053 */
1054int
1055i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1056 struct drm_file *file_priv)
1057{
a09ba7fa 1058 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 1059 struct drm_i915_gem_set_domain *args = data;
1060 struct drm_gem_object *obj;
652c393a 1061 struct drm_i915_gem_object *obj_priv;
2ef7eeaa
EA		 1062 uint32_t read_domains = args->read_domains;
1063 uint32_t write_domain = args->write_domain;
673a394b
EA		 1064 int ret;
1065
1066 if (!(dev->driver->driver_features & DRIVER_GEM))
1067 return -ENODEV;
1068
2ef7eeaa 1069 /* Only handle setting domains to types used by the CPU. */
21d509e3 1070 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1071 return -EINVAL;
1072
21d509e3 1073 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1074 return -EINVAL;
1075
1076 /* Having something in the write domain implies it's in the read
1077 * domain, and only that read domain. Enforce that in the request.
1078 */
1079 if (write_domain != 0 && read_domains != write_domain)
1080 return -EINVAL;
1081
673a394b
EA		 1082 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1083 if (obj == NULL)
bf79cb91 1084 return -ENOENT;
23010e43 1085 obj_priv = to_intel_bo(obj);
673a394b 1086
76c1dec1
CW		 1087 ret = i915_mutex_lock_interruptible(dev);
1088 if (ret) {
1089 drm_gem_object_unreference_unlocked(obj);
1090 return ret;
1091 }
652c393a
JB		 1092
1093 intel_mark_busy(dev, obj);
1094
673a394b 1095#if WATCH_BUF
cfd43c02 1096 DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
2ef7eeaa 1097 obj, obj->size, read_domains, write_domain);
673a394b 1098#endif
2ef7eeaa
EA		 1099 if (read_domains & I915_GEM_DOMAIN_GTT) {
1100 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392 1101
a09ba7fa
EA		 1102 /* Update the LRU on the fence for the CPU access that's
1103 * about to occur.
1104 */
1105 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
007cc8ac
DV		 1106 struct drm_i915_fence_reg *reg =
1107 &dev_priv->fence_regs[obj_priv->fence_reg];
1108 list_move_tail(&reg->lru_list,
a09ba7fa
EA		 1109 &dev_priv->mm.fence_list);
1110 }
1111
02354392
EA		 1112 /* Silently promote "you're not bound, there was nothing to do"
1113 * to success, since the client was just asking us to
1114 * make sure everything was done.
1115 */
1116 if (ret == -EINVAL)
1117 ret = 0;
2ef7eeaa 1118 } else {
e47c68e9 1119 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1120 }
1121
7d1c4804
CW		 1122 /* Maintain LRU order of "inactive" objects */
1123 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1124 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1125
673a394b
EA		 1126 drm_gem_object_unreference(obj);
1127 mutex_unlock(&dev->struct_mutex);
1128 return ret;
1129}
1130
1131/**
1132 * Called when user space has done writes to this buffer
1133 */
1134int
1135i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1136 struct drm_file *file_priv)
1137{
1138 struct drm_i915_gem_sw_finish *args = data;
1139 struct drm_gem_object *obj;
1140 struct drm_i915_gem_object *obj_priv;
1141 int ret = 0;
1142
1143 if (!(dev->driver->driver_features & DRIVER_GEM))
1144 return -ENODEV;
1145
673a394b 1146 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
76c1dec1 1147 if (obj == NULL)
bf79cb91 1148 return -ENOENT;
76c1dec1
CW		 1149
1150 ret = i915_mutex_lock_interruptible(dev);
1151 if (ret) {
1152 drm_gem_object_unreference_unlocked(obj);
1153 return ret;
673a394b
EA		 1154 }
1155
1156#if WATCH_BUF
cfd43c02 1157 DRM_INFO("%s: sw_finish %d (%p %zd)\n",
673a394b
EA		 1158 __func__, args->handle, obj, obj->size);
1159#endif
23010e43 1160 obj_priv = to_intel_bo(obj);
673a394b
EA		 1161
1162 /* Pinned buffers may be scanout, so flush the cache */
e47c68e9
EA		 1163 if (obj_priv->pin_count)
1164 i915_gem_object_flush_cpu_write_domain(obj);
1165
673a394b
EA		 1166 drm_gem_object_unreference(obj);
1167 mutex_unlock(&dev->struct_mutex);
1168 return ret;
1169}
1170
1171/**
1172 * Maps the contents of an object, returning the address it is mapped
1173 * into.
1174 *
1175 * While the mapping holds a reference on the contents of the object, it doesn't
1176 * imply a ref on the object itself.
1177 */
1178int
1179i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1180 struct drm_file *file_priv)
1181{
1182 struct drm_i915_gem_mmap *args = data;
1183 struct drm_gem_object *obj;
1184 loff_t offset;
1185 unsigned long addr;
1186
1187 if (!(dev->driver->driver_features & DRIVER_GEM))
1188 return -ENODEV;
1189
1190 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1191 if (obj == NULL)
bf79cb91 1192 return -ENOENT;
673a394b
EA		 1193
1194 offset = args->offset;
1195
1196 down_write(&current->mm->mmap_sem);
1197 addr = do_mmap(obj->filp, 0, args->size,
1198 PROT_READ | PROT_WRITE, MAP_SHARED,
1199 args->offset);
1200 up_write(&current->mm->mmap_sem);
bc9025bd 1201 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1202 if (IS_ERR((void *)addr))
1203 return addr;
1204
1205 args->addr_ptr = (uint64_t) addr;
1206
1207 return 0;
1208}
1209
de151cf6
JB		 1210/**
1211 * i915_gem_fault - fault a page into the GTT
1212 * vma: VMA in question
1213 * vmf: fault info
1214 *
1215 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1216 * from userspace. The fault handler takes care of binding the object to
1217 * the GTT (if needed), allocating and programming a fence register (again,
1218 * only if needed based on whether the old reg is still valid or the object
1219 * is tiled) and inserting a new PTE into the faulting process.
1220 *
1221 * Note that the faulting process may involve evicting existing objects
1222 * from the GTT and/or fence registers to make room. So performance may
1223 * suffer if the GTT working set is large or there are few fence registers
1224 * left.
1225 */
1226int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1227{
1228 struct drm_gem_object *obj = vma->vm_private_data;
1229 struct drm_device *dev = obj->dev;
7d1c4804 1230 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1231 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 1232 pgoff_t page_offset;
1233 unsigned long pfn;
1234 int ret = 0;
0f973f27 1235 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1236
1237 /* We don't use vmf->pgoff since that has the fake offset */
1238 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1239 PAGE_SHIFT;
1240
1241 /* Now bind it into the GTT if needed */
1242 mutex_lock(&dev->struct_mutex);
1243 if (!obj_priv->gtt_space) {
e67b8ce1 1244 ret = i915_gem_object_bind_to_gtt(obj, 0);
c715089f
CW		 1245 if (ret)
1246 goto unlock;
07f4f3e8 1247
07f4f3e8 1248 ret = i915_gem_object_set_to_gtt_domain(obj, write);
c715089f
CW		 1249 if (ret)
1250 goto unlock;
de151cf6
JB		 1251 }
1252
1253 /* Need a new fence register? */
a09ba7fa 1254 if (obj_priv->tiling_mode != I915_TILING_NONE) {
2cf34d7b 1255 ret = i915_gem_object_get_fence_reg(obj, true);
c715089f
CW		 1256 if (ret)
1257 goto unlock;
d9ddcb96 1258 }
de151cf6 1259
7d1c4804
CW		 1260 if (i915_gem_object_is_inactive(obj_priv))
1261 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1262
de151cf6
JB		 1263 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1264 page_offset;
1265
1266 /* Finally, remap it using the new GTT offset */
1267 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1268unlock:
de151cf6
JB		 1269 mutex_unlock(&dev->struct_mutex);
1270
1271 switch (ret) {
c715089f
CW		 1272 case 0:
1273 case -ERESTARTSYS:
1274 return VM_FAULT_NOPAGE;
de151cf6
JB		 1275 case -ENOMEM:
1276 case -EAGAIN:
1277 return VM_FAULT_OOM;
de151cf6 1278 default:
c715089f 1279 return VM_FAULT_SIGBUS;
de151cf6
JB		 1280 }
1281}
1282
1283/**
1284 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1285 * @obj: obj in question
1286 *
1287 * GEM memory mapping works by handing back to userspace a fake mmap offset
1288 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1289 * up the object based on the offset and sets up the various memory mapping
1290 * structures.
1291 *
1292 * This routine allocates and attaches a fake offset for @obj.
1293 */
1294static int
1295i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1296{
1297 struct drm_device *dev = obj->dev;
1298 struct drm_gem_mm *mm = dev->mm_private;
23010e43 1299 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 1300 struct drm_map_list *list;
f77d390c 1301 struct drm_local_map *map;
de151cf6
JB		 1302 int ret = 0;
1303
1304 /* Set the object up for mmap'ing */
1305 list = &obj->map_list;
9a298b2a 1306 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
de151cf6
JB		 1307 if (!list->map)
1308 return -ENOMEM;
1309
1310 map = list->map;
1311 map->type = _DRM_GEM;
1312 map->size = obj->size;
1313 map->handle = obj;
1314
1315 /* Get a DRM GEM mmap offset allocated... */
1316 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1317 obj->size / PAGE_SIZE, 0, 0);
1318 if (!list->file_offset_node) {
1319 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
9e0ae534 1320 ret = -ENOSPC;
de151cf6
JB		 1321 goto out_free_list;
1322 }
1323
1324 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1325 obj->size / PAGE_SIZE, 0);
1326 if (!list->file_offset_node) {
1327 ret = -ENOMEM;
1328 goto out_free_list;
1329 }
1330
1331 list->hash.key = list->file_offset_node->start;
9e0ae534
CW		 1332 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1333 if (ret) {
de151cf6
JB		 1334 DRM_ERROR("failed to add to map hash\n");
1335 goto out_free_mm;
1336 }
1337
1338 /* By now we should be all set, any drm_mmap request on the offset
1339 * below will get to our mmap & fault handler */
1340 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1341
1342 return 0;
1343
1344out_free_mm:
1345 drm_mm_put_block(list->file_offset_node);
1346out_free_list:
9a298b2a 1347 kfree(list->map);
de151cf6
JB		 1348
1349 return ret;
1350}
1351
901782b2
CW		 1352/**
1353 * i915_gem_release_mmap - remove physical page mappings
1354 * @obj: obj in question
1355 *
af901ca1 1356 * Preserve the reservation of the mmapping with the DRM core code, but
901782b2
CW		 1357 * relinquish ownership of the pages back to the system.
1358 *
1359 * It is vital that we remove the page mapping if we have mapped a tiled
1360 * object through the GTT and then lose the fence register due to
1361 * resource pressure. Similarly if the object has been moved out of the
1362 * aperture, than pages mapped into userspace must be revoked. Removing the
1363 * mapping will then trigger a page fault on the next user access, allowing
1364 * fixup by i915_gem_fault().
1365 */
d05ca301 1366void
901782b2
CW		 1367i915_gem_release_mmap(struct drm_gem_object *obj)
1368{
1369 struct drm_device *dev = obj->dev;
23010e43 1370 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
901782b2
CW		 1371
1372 if (dev->dev_mapping)
1373 unmap_mapping_range(dev->dev_mapping,
1374 obj_priv->mmap_offset, obj->size, 1);
1375}
1376
ab00b3e5
JB		 1377static void
1378i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1379{
1380 struct drm_device *dev = obj->dev;
23010e43 1381 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ab00b3e5
JB		 1382 struct drm_gem_mm *mm = dev->mm_private;
1383 struct drm_map_list *list;
1384
1385 list = &obj->map_list;
1386 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1387
1388 if (list->file_offset_node) {
1389 drm_mm_put_block(list->file_offset_node);
1390 list->file_offset_node = NULL;
1391 }
1392
1393 if (list->map) {
9a298b2a 1394 kfree(list->map);
ab00b3e5
JB		 1395 list->map = NULL;
1396 }
1397
1398 obj_priv->mmap_offset = 0;
1399}
1400
de151cf6
JB		 1401/**
1402 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1403 * @obj: object to check
1404 *
1405 * Return the required GTT alignment for an object, taking into account
1406 * potential fence register mapping if needed.
1407 */
1408static uint32_t
1409i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1410{
1411 struct drm_device *dev = obj->dev;
23010e43 1412 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 1413 int start, i;
1414
1415 /*
1416 * Minimum alignment is 4k (GTT page size), but might be greater
1417 * if a fence register is needed for the object.
1418 */
a6c45cf0 1419 if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
de151cf6
JB		 1420 return 4096;
1421
1422 /*
1423 * Previous chips need to be aligned to the size of the smallest
1424 * fence register that can contain the object.
1425 */
a6c45cf0 1426 if (INTEL_INFO(dev)->gen == 3)
de151cf6
JB		 1427 start = 1024*1024;
1428 else
1429 start = 512*1024;
1430
1431 for (i = start; i < obj->size; i <<= 1)
1432 ;
1433
1434 return i;
1435}
1436
1437/**
1438 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1439 * @dev: DRM device
1440 * @data: GTT mapping ioctl data
1441 * @file_priv: GEM object info
1442 *
1443 * Simply returns the fake offset to userspace so it can mmap it.
1444 * The mmap call will end up in drm_gem_mmap(), which will set things
1445 * up so we can get faults in the handler above.
1446 *
1447 * The fault handler will take care of binding the object into the GTT
1448 * (since it may have been evicted to make room for something), allocating
1449 * a fence register, and mapping the appropriate aperture address into
1450 * userspace.
1451 */
1452int
1453i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1454 struct drm_file *file_priv)
1455{
1456 struct drm_i915_gem_mmap_gtt *args = data;
de151cf6
JB		 1457 struct drm_gem_object *obj;
1458 struct drm_i915_gem_object *obj_priv;
1459 int ret;
1460
1461 if (!(dev->driver->driver_features & DRIVER_GEM))
1462 return -ENODEV;
1463
1464 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1465 if (obj == NULL)
bf79cb91 1466 return -ENOENT;
de151cf6 1467
76c1dec1
CW		 1468 ret = i915_mutex_lock_interruptible(dev);
1469 if (ret) {
1470 drm_gem_object_unreference_unlocked(obj);
1471 return ret;
1472 }
de151cf6 1473
23010e43 1474 obj_priv = to_intel_bo(obj);
de151cf6 1475
ab18282d
CW		 1476 if (obj_priv->madv != I915_MADV_WILLNEED) {
1477 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1478 drm_gem_object_unreference(obj);
1479 mutex_unlock(&dev->struct_mutex);
1480 return -EINVAL;
1481 }
1482
1483
de151cf6
JB		 1484 if (!obj_priv->mmap_offset) {
1485 ret = i915_gem_create_mmap_offset(obj);
13af1062
CW		 1486 if (ret) {
1487 drm_gem_object_unreference(obj);
1488 mutex_unlock(&dev->struct_mutex);
de151cf6 1489 return ret;
13af1062 1490 }
de151cf6
JB		 1491 }
1492
1493 args->offset = obj_priv->mmap_offset;
1494
de151cf6
JB		 1495 /*
1496 * Pull it into the GTT so that we have a page list (makes the
1497 * initial fault faster and any subsequent flushing possible).
1498 */
1499 if (!obj_priv->agp_mem) {
e67b8ce1 1500 ret = i915_gem_object_bind_to_gtt(obj, 0);
de151cf6
JB		 1501 if (ret) {
1502 drm_gem_object_unreference(obj);
1503 mutex_unlock(&dev->struct_mutex);
1504 return ret;
1505 }
de151cf6
JB		 1506 }
1507
1508 drm_gem_object_unreference(obj);
1509 mutex_unlock(&dev->struct_mutex);
1510
1511 return 0;
1512}
1513
6911a9b8 1514void
856fa198 1515i915_gem_object_put_pages(struct drm_gem_object *obj)
673a394b 1516{
23010e43 1517 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 1518 int page_count = obj->size / PAGE_SIZE;
1519 int i;
1520
856fa198 1521 BUG_ON(obj_priv->pages_refcount == 0);
bb6baf76 1522 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
673a394b 1523
856fa198
EA		 1524 if (--obj_priv->pages_refcount != 0)
1525 return;
673a394b 1526
280b713b
EA		 1527 if (obj_priv->tiling_mode != I915_TILING_NONE)
1528 i915_gem_object_save_bit_17_swizzle(obj);
1529
3ef94daa 1530 if (obj_priv->madv == I915_MADV_DONTNEED)
13a05fd9 1531 obj_priv->dirty = 0;
3ef94daa
CW		 1532
1533 for (i = 0; i < page_count; i++) {
3ef94daa
CW		 1534 if (obj_priv->dirty)
1535 set_page_dirty(obj_priv->pages[i]);
1536
1537 if (obj_priv->madv == I915_MADV_WILLNEED)
856fa198 1538 mark_page_accessed(obj_priv->pages[i]);
3ef94daa
CW		 1539
1540 page_cache_release(obj_priv->pages[i]);
1541 }
673a394b
EA		 1542 obj_priv->dirty = 0;
1543
8e7d2b2c 1544 drm_free_large(obj_priv->pages);
856fa198 1545 obj_priv->pages = NULL;
673a394b
EA		 1546}
1547
1548static void
617dbe27 1549i915_gem_object_move_to_active(struct drm_gem_object *obj,
852835f3 1550 struct intel_ring_buffer *ring)
673a394b 1551{
5c12a07e 1552 struct drm_i915_private *dev_priv = obj->dev->dev_private;
23010e43 1553 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
617dbe27 1554
852835f3
ZN		 1555 BUG_ON(ring == NULL);
1556 obj_priv->ring = ring;
673a394b
EA		 1557
1558 /* Add a reference if we're newly entering the active list. */
1559 if (!obj_priv->active) {
1560 drm_gem_object_reference(obj);
1561 obj_priv->active = 1;
1562 }
e35a41de 1563
673a394b 1564 /* Move from whatever list we were on to the tail of execution. */
852835f3 1565 list_move_tail(&obj_priv->list, &ring->active_list);
5c12a07e 1566 obj_priv->last_rendering_seqno = dev_priv->next_seqno;
673a394b
EA		 1567}
1568
ce44b0ea
EA		 1569static void
1570i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1571{
1572 struct drm_device *dev = obj->dev;
1573 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1574 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ce44b0ea
EA		 1575
1576 BUG_ON(!obj_priv->active);
1577 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1578 obj_priv->last_rendering_seqno = 0;
1579}
673a394b 1580
963b4836
CW		 1581/* Immediately discard the backing storage */
1582static void
1583i915_gem_object_truncate(struct drm_gem_object *obj)
1584{
23010e43 1585 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
bb6baf76 1586 struct inode *inode;
963b4836 1587
ae9fed6b
CW		 1588 /* Our goal here is to return as much of the memory as
1589 * is possible back to the system as we are called from OOM.
1590 * To do this we must instruct the shmfs to drop all of its
1591 * backing pages, *now*. Here we mirror the actions taken
1592 * when by shmem_delete_inode() to release the backing store.
1593 */
bb6baf76 1594 inode = obj->filp->f_path.dentry->d_inode;
ae9fed6b
CW		 1595 truncate_inode_pages(inode->i_mapping, 0);
1596 if (inode->i_op->truncate_range)
1597 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
bb6baf76
CW		 1598
1599 obj_priv->madv = __I915_MADV_PURGED;
963b4836
CW		 1600}
1601
1602static inline int
1603i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1604{
1605 return obj_priv->madv == I915_MADV_DONTNEED;
1606}
1607
673a394b
EA		 1608static void
1609i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1610{
1611 struct drm_device *dev = obj->dev;
1612 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1613 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 1614
1615 i915_verify_inactive(dev, __FILE__, __LINE__);
1616 if (obj_priv->pin_count != 0)
f13d3f73 1617 list_move_tail(&obj_priv->list, &dev_priv->mm.pinned_list);
673a394b
EA		 1618 else
1619 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1620
99fcb766
DV		 1621 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1622
ce44b0ea 1623 obj_priv->last_rendering_seqno = 0;
852835f3 1624 obj_priv->ring = NULL;
673a394b
EA		 1625 if (obj_priv->active) {
1626 obj_priv->active = 0;
1627 drm_gem_object_unreference(obj);
1628 }
1629 i915_verify_inactive(dev, __FILE__, __LINE__);
1630}
1631
9220434a 1632static void
63560396 1633i915_gem_process_flushing_list(struct drm_device *dev,
8a1a49f9 1634 uint32_t flush_domains,
852835f3 1635 struct intel_ring_buffer *ring)
63560396
DV		 1636{
1637 drm_i915_private_t *dev_priv = dev->dev_private;
1638 struct drm_i915_gem_object *obj_priv, *next;
1639
1640 list_for_each_entry_safe(obj_priv, next,
1641 &dev_priv->mm.gpu_write_list,
1642 gpu_write_list) {
a8089e84 1643 struct drm_gem_object *obj = &obj_priv->base;
63560396 1644
2b6efaa4
CW		 1645 if (obj->write_domain & flush_domains &&
1646 obj_priv->ring == ring) {
63560396
DV		 1647 uint32_t old_write_domain = obj->write_domain;
1648
1649 obj->write_domain = 0;
1650 list_del_init(&obj_priv->gpu_write_list);
617dbe27 1651 i915_gem_object_move_to_active(obj, ring);
63560396
DV		 1652
1653 /* update the fence lru list */
007cc8ac
DV		 1654 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1655 struct drm_i915_fence_reg *reg =
1656 &dev_priv->fence_regs[obj_priv->fence_reg];
1657 list_move_tail(&reg->lru_list,
63560396 1658 &dev_priv->mm.fence_list);
007cc8ac 1659 }
63560396
DV		 1660
1661 trace_i915_gem_object_change_domain(obj,
1662 obj->read_domains,
1663 old_write_domain);
1664 }
1665 }
1666}
8187a2b7 1667
5a5a0c64 1668uint32_t
8a1a49f9 1669i915_add_request(struct drm_device *dev,
f787a5f5 1670 struct drm_file *file,
8dc5d147 1671 struct drm_i915_gem_request *request,
8a1a49f9 1672 struct intel_ring_buffer *ring)
673a394b
EA		 1673{
1674 drm_i915_private_t *dev_priv = dev->dev_private;
f787a5f5 1675 struct drm_i915_file_private *file_priv = NULL;
673a394b
EA		 1676 uint32_t seqno;
1677 int was_empty;
673a394b 1678
f787a5f5
CW		 1679 if (file != NULL)
1680 file_priv = file->driver_priv;
b962442e 1681
8dc5d147
CW		 1682 if (request == NULL) {
1683 request = kzalloc(sizeof(*request), GFP_KERNEL);
1684 if (request == NULL)
1685 return 0;
1686 }
673a394b 1687
f787a5f5 1688 seqno = ring->add_request(dev, ring, 0);
673a394b
EA		 1689
1690 request->seqno = seqno;
852835f3 1691 request->ring = ring;
673a394b 1692 request->emitted_jiffies = jiffies;
852835f3
ZN		 1693 was_empty = list_empty(&ring->request_list);
1694 list_add_tail(&request->list, &ring->request_list);
1695
f787a5f5
CW		 1696 if (file_priv) {
1697 mutex_lock(&file_priv->mutex);
1698 request->file_priv = file_priv;
b962442e 1699 list_add_tail(&request->client_list,
f787a5f5
CW		 1700 &file_priv->mm.request_list);
1701 mutex_unlock(&file_priv->mutex);
b962442e 1702 }
673a394b 1703
f65d9421 1704 if (!dev_priv->mm.suspended) {
b3b079db
CW		 1705 mod_timer(&dev_priv->hangcheck_timer,
1706 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
f65d9421 1707 if (was_empty)
b3b079db
CW		 1708 queue_delayed_work(dev_priv->wq,
1709 &dev_priv->mm.retire_work, HZ);
f65d9421 1710 }
673a394b
EA		 1711 return seqno;
1712}
1713
1714/**
1715 * Command execution barrier
1716 *
1717 * Ensures that all commands in the ring are finished
1718 * before signalling the CPU
1719 */
8a1a49f9 1720static void
852835f3 1721i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
673a394b 1722{
673a394b 1723 uint32_t flush_domains = 0;
673a394b
EA		 1724
1725 /* The sampler always gets flushed on i965 (sigh) */
a6c45cf0 1726 if (INTEL_INFO(dev)->gen >= 4)
673a394b 1727 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
852835f3
ZN		 1728
1729 ring->flush(dev, ring,
1730 I915_GEM_DOMAIN_COMMAND, flush_domains);
673a394b
EA		 1731}
1732
f787a5f5
CW		 1733static inline void
1734i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
673a394b 1735{
f787a5f5
CW		 1736 if (request->file_priv) {
1737 mutex_lock(&request->file_priv->mutex);
1738 list_del(&request->client_list);
1739 mutex_unlock(&request->file_priv->mutex);
1740 }
673a394b
EA		 1741}
1742
dfaae392
CW		 1743static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1744 struct intel_ring_buffer *ring)
9375e446 1745{
dfaae392
CW		 1746 while (!list_empty(&ring->request_list)) {
1747 struct drm_i915_gem_request *request;
9375e446 1748
dfaae392
CW		 1749 request = list_first_entry(&ring->request_list,
1750 struct drm_i915_gem_request,
1751 list);
1752
1753 list_del(&request->list);
f787a5f5 1754 i915_gem_request_remove_from_client(request);
dfaae392
CW		 1755 kfree(request);
1756 }
1757
1758 while (!list_empty(&ring->active_list)) {
9375e446
CW		 1759 struct drm_i915_gem_object *obj_priv;
1760
dfaae392 1761 obj_priv = list_first_entry(&ring->active_list,
9375e446
CW		 1762 struct drm_i915_gem_object,
1763 list);
1764
1765 obj_priv->base.write_domain = 0;
dfaae392 1766 list_del_init(&obj_priv->gpu_write_list);
9375e446
CW		 1767 i915_gem_object_move_to_inactive(&obj_priv->base);
1768 }
1769}
1770
dfaae392 1771void i915_gem_reset_lists(struct drm_device *dev)
77f01230
CW		 1772{
1773 struct drm_i915_private *dev_priv = dev->dev_private;
1774 struct drm_i915_gem_object *obj_priv;
1775
dfaae392
CW		 1776 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1777 if (HAS_BSD(dev))
1778 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1779
1780 /* Remove anything from the flushing lists. The GPU cache is likely
1781 * to be lost on reset along with the data, so simply move the
1782 * lost bo to the inactive list.
1783 */
1784 while (!list_empty(&dev_priv->mm.flushing_list)) {
1785 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1786 struct drm_i915_gem_object,
1787 list);
1788
1789 obj_priv->base.write_domain = 0;
1790 list_del_init(&obj_priv->gpu_write_list);
1791 i915_gem_object_move_to_inactive(&obj_priv->base);
1792 }
1793
1794 /* Move everything out of the GPU domains to ensure we do any
1795 * necessary invalidation upon reuse.
1796 */
77f01230
CW		 1797 list_for_each_entry(obj_priv,
1798 &dev_priv->mm.inactive_list,
1799 list)
1800 {
1801 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1802 }
1803}
1804
673a394b
EA		 1805/**
1806 * This function clears the request list as sequence numbers are passed.
1807 */
b09a1fec
CW		 1808static void
1809i915_gem_retire_requests_ring(struct drm_device *dev,
1810 struct intel_ring_buffer *ring)
673a394b
EA		 1811{
1812 drm_i915_private_t *dev_priv = dev->dev_private;
1813 uint32_t seqno;
1814
b84d5f0c
CW		 1815 if (!ring->status_page.page_addr ||
1816 list_empty(&ring->request_list))
6c0594a3
KW		 1817 return;
1818
f787a5f5 1819 seqno = ring->get_seqno(dev, ring);
852835f3 1820 while (!list_empty(&ring->request_list)) {
673a394b 1821 struct drm_i915_gem_request *request;
673a394b 1822
852835f3 1823 request = list_first_entry(&ring->request_list,
673a394b
EA		 1824 struct drm_i915_gem_request,
1825 list);
673a394b 1826
dfaae392 1827 if (!i915_seqno_passed(seqno, request->seqno))
b84d5f0c
CW		 1828 break;
1829
1830 trace_i915_gem_request_retire(dev, request->seqno);
1831
1832 list_del(&request->list);
f787a5f5 1833 i915_gem_request_remove_from_client(request);
b84d5f0c
CW		 1834 kfree(request);
1835 }
1836
1837 /* Move any buffers on the active list that are no longer referenced
1838 * by the ringbuffer to the flushing/inactive lists as appropriate.
1839 */
1840 while (!list_empty(&ring->active_list)) {
1841 struct drm_gem_object *obj;
1842 struct drm_i915_gem_object *obj_priv;
1843
1844 obj_priv = list_first_entry(&ring->active_list,
1845 struct drm_i915_gem_object,
1846 list);
673a394b 1847
dfaae392 1848 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
673a394b 1849 break;
b84d5f0c
CW		 1850
1851 obj = &obj_priv->base;
1852
1853#if WATCH_LRU
1854 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1855 __func__, request->seqno, obj);
1856#endif
1857
1858 if (obj->write_domain != 0)
1859 i915_gem_object_move_to_flushing(obj);
1860 else
1861 i915_gem_object_move_to_inactive(obj);
673a394b 1862 }
9d34e5db
CW		 1863
1864 if (unlikely (dev_priv->trace_irq_seqno &&
1865 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
8187a2b7 1866 ring->user_irq_put(dev, ring);
9d34e5db
CW		 1867 dev_priv->trace_irq_seqno = 0;
1868 }
673a394b
EA		 1869}
1870
b09a1fec
CW		 1871void
1872i915_gem_retire_requests(struct drm_device *dev)
1873{
1874 drm_i915_private_t *dev_priv = dev->dev_private;
1875
be72615b
CW		 1876 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1877 struct drm_i915_gem_object *obj_priv, *tmp;
1878
1879 /* We must be careful that during unbind() we do not
1880 * accidentally infinitely recurse into retire requests.
1881 * Currently:
1882 * retire -> free -> unbind -> wait -> retire_ring
1883 */
1884 list_for_each_entry_safe(obj_priv, tmp,
1885 &dev_priv->mm.deferred_free_list,
1886 list)
1887 i915_gem_free_object_tail(&obj_priv->base);
1888 }
1889
b09a1fec
CW		 1890 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1891 if (HAS_BSD(dev))
1892 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1893}
1894
75ef9da2 1895static void
673a394b
EA		 1896i915_gem_retire_work_handler(struct work_struct *work)
1897{
1898 drm_i915_private_t *dev_priv;
1899 struct drm_device *dev;
1900
1901 dev_priv = container_of(work, drm_i915_private_t,
1902 mm.retire_work.work);
1903 dev = dev_priv->dev;
1904
1905 mutex_lock(&dev->struct_mutex);
b09a1fec 1906 i915_gem_retire_requests(dev);
d1b851fc 1907
6dbe2772 1908 if (!dev_priv->mm.suspended &&
d1b851fc
ZN		 1909 (!list_empty(&dev_priv->render_ring.request_list) ||
1910 (HAS_BSD(dev) &&
1911 !list_empty(&dev_priv->bsd_ring.request_list))))
9c9fe1f8 1912 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
673a394b
EA		 1913 mutex_unlock(&dev->struct_mutex);
1914}
1915
5a5a0c64 1916int
852835f3 1917i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
8a1a49f9 1918 bool interruptible, struct intel_ring_buffer *ring)
673a394b
EA		 1919{
1920 drm_i915_private_t *dev_priv = dev->dev_private;
802c7eb6 1921 u32 ier;
673a394b
EA		 1922 int ret = 0;
1923
1924 BUG_ON(seqno == 0);
1925
30dbf0c0
CW		 1926 if (atomic_read(&dev_priv->mm.wedged))
1927 return -EAGAIN;
1928
e35a41de 1929 if (seqno == dev_priv->next_seqno) {
8dc5d147 1930 seqno = i915_add_request(dev, NULL, NULL, ring);
e35a41de
DV		 1931 if (seqno == 0)
1932 return -ENOMEM;
1933 }
1934
f787a5f5 1935 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
bad720ff 1936 if (HAS_PCH_SPLIT(dev))
036a4a7d
ZW		 1937 ier = I915_READ(DEIER) | I915_READ(GTIER);
1938 else
1939 ier = I915_READ(IER);
802c7eb6
JB		 1940 if (!ier) {
1941 DRM_ERROR("something (likely vbetool) disabled "
1942 "interrupts, re-enabling\n");
1943 i915_driver_irq_preinstall(dev);
1944 i915_driver_irq_postinstall(dev);
1945 }
1946
1c5d22f7
CW		 1947 trace_i915_gem_request_wait_begin(dev, seqno);
1948
852835f3 1949 ring->waiting_gem_seqno = seqno;
8187a2b7 1950 ring->user_irq_get(dev, ring);
48764bf4 1951 if (interruptible)
852835f3
ZN		 1952 ret = wait_event_interruptible(ring->irq_queue,
1953 i915_seqno_passed(
f787a5f5 1954 ring->get_seqno(dev, ring), seqno)
852835f3 1955 || atomic_read(&dev_priv->mm.wedged));
48764bf4 1956 else
852835f3
ZN		 1957 wait_event(ring->irq_queue,
1958 i915_seqno_passed(
f787a5f5 1959 ring->get_seqno(dev, ring), seqno)
852835f3 1960 || atomic_read(&dev_priv->mm.wedged));
48764bf4 1961
8187a2b7 1962 ring->user_irq_put(dev, ring);
852835f3 1963 ring->waiting_gem_seqno = 0;
1c5d22f7
CW		 1964
1965 trace_i915_gem_request_wait_end(dev, seqno);
673a394b 1966 }
ba1234d1 1967 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0 1968 ret = -EAGAIN;
673a394b
EA		 1969
1970 if (ret && ret != -ERESTARTSYS)
8bff917c 1971 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
f787a5f5 1972 __func__, ret, seqno, ring->get_seqno(dev, ring),
8bff917c 1973 dev_priv->next_seqno);
673a394b
EA		 1974
1975 /* Directly dispatch request retiring. While we have the work queue
1976 * to handle this, the waiter on a request often wants an associated
1977 * buffer to have made it to the inactive list, and we would need
1978 * a separate wait queue to handle that.
1979 */
1980 if (ret == 0)
b09a1fec 1981 i915_gem_retire_requests_ring(dev, ring);
673a394b
EA		 1982
1983 return ret;
1984}
1985
48764bf4
DV		 1986/**
1987 * Waits for a sequence number to be signaled, and cleans up the
1988 * request and object lists appropriately for that event.
1989 */
1990static int
852835f3
ZN		 1991i915_wait_request(struct drm_device *dev, uint32_t seqno,
1992 struct intel_ring_buffer *ring)
48764bf4 1993{
852835f3 1994 return i915_do_wait_request(dev, seqno, 1, ring);
48764bf4
DV		 1995}
1996
20f0cd55 1997static void
9220434a 1998i915_gem_flush_ring(struct drm_device *dev,
c78ec30b 1999 struct drm_file *file_priv,
9220434a
CW		 2000 struct intel_ring_buffer *ring,
2001 uint32_t invalidate_domains,
2002 uint32_t flush_domains)
2003{
2004 ring->flush(dev, ring, invalidate_domains, flush_domains);
2005 i915_gem_process_flushing_list(dev, flush_domains, ring);
2006}
2007
8187a2b7
ZN		 2008static void
2009i915_gem_flush(struct drm_device *dev,
c78ec30b 2010 struct drm_file *file_priv,
8187a2b7 2011 uint32_t invalidate_domains,
9220434a
CW		 2012 uint32_t flush_domains,
2013 uint32_t flush_rings)
8187a2b7
ZN		 2014{
2015 drm_i915_private_t *dev_priv = dev->dev_private;
8bff917c 2016
8187a2b7
ZN		 2017 if (flush_domains & I915_GEM_DOMAIN_CPU)
2018 drm_agp_chipset_flush(dev);
8bff917c 2019
9220434a
CW		 2020 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2021 if (flush_rings & RING_RENDER)
c78ec30b 2022 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 2023 &dev_priv->render_ring,
2024 invalidate_domains, flush_domains);
2025 if (flush_rings & RING_BSD)
c78ec30b 2026 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 2027 &dev_priv->bsd_ring,
2028 invalidate_domains, flush_domains);
2029 }
8187a2b7
ZN		 2030}
2031
673a394b
EA		 2032/**
2033 * Ensures that all rendering to the object has completed and the object is
2034 * safe to unbind from the GTT or access from the CPU.
2035 */
2036static int
2cf34d7b
CW		 2037i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2038 bool interruptible)
673a394b
EA		 2039{
2040 struct drm_device *dev = obj->dev;
23010e43 2041 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2042 int ret;
2043
e47c68e9
EA		 2044 /* This function only exists to support waiting for existing rendering,
2045 * not for emitting required flushes.
673a394b 2046 */
e47c68e9 2047 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 2048
2049 /* If there is rendering queued on the buffer being evicted, wait for
2050 * it.
2051 */
2052 if (obj_priv->active) {
2053#if WATCH_BUF
2054 DRM_INFO("%s: object %p wait for seqno %08x\n",
2055 __func__, obj, obj_priv->last_rendering_seqno);
2056#endif
2cf34d7b
CW		 2057 ret = i915_do_wait_request(dev,
2058 obj_priv->last_rendering_seqno,
2059 interruptible,
2060 obj_priv->ring);
2061 if (ret)
673a394b
EA		 2062 return ret;
2063 }
2064
2065 return 0;
2066}
2067
2068/**
2069 * Unbinds an object from the GTT aperture.
2070 */
0f973f27 2071int
673a394b
EA		 2072i915_gem_object_unbind(struct drm_gem_object *obj)
2073{
2074 struct drm_device *dev = obj->dev;
23010e43 2075 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2076 int ret = 0;
2077
2078#if WATCH_BUF
2079 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
2080 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
2081#endif
2082 if (obj_priv->gtt_space == NULL)
2083 return 0;
2084
2085 if (obj_priv->pin_count != 0) {
2086 DRM_ERROR("Attempting to unbind pinned buffer\n");
2087 return -EINVAL;
2088 }
2089
5323fd04
EA		 2090 /* blow away mappings if mapped through GTT */
2091 i915_gem_release_mmap(obj);
2092
673a394b
EA		 2093 /* Move the object to the CPU domain to ensure that
2094 * any possible CPU writes while it's not in the GTT
2095 * are flushed when we go to remap it. This will
2096 * also ensure that all pending GPU writes are finished
2097 * before we unbind.
2098 */
e47c68e9 2099 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
8dc1775d 2100 if (ret == -ERESTARTSYS)
673a394b 2101 return ret;
8dc1775d
CW		 2102 /* Continue on if we fail due to EIO, the GPU is hung so we
2103 * should be safe and we need to cleanup or else we might
2104 * cause memory corruption through use-after-free.
2105 */
673a394b 2106
96b47b65
DV		 2107 /* release the fence reg _after_ flushing */
2108 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2109 i915_gem_clear_fence_reg(obj);
2110
673a394b
EA		 2111 if (obj_priv->agp_mem != NULL) {
2112 drm_unbind_agp(obj_priv->agp_mem);
2113 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2114 obj_priv->agp_mem = NULL;
2115 }
2116
856fa198 2117 i915_gem_object_put_pages(obj);
a32808c0 2118 BUG_ON(obj_priv->pages_refcount);
673a394b
EA		 2119
2120 if (obj_priv->gtt_space) {
2121 atomic_dec(&dev->gtt_count);
2122 atomic_sub(obj->size, &dev->gtt_memory);
2123
2124 drm_mm_put_block(obj_priv->gtt_space);
2125 obj_priv->gtt_space = NULL;
2126 }
2127
f13d3f73 2128 list_del_init(&obj_priv->list);
673a394b 2129
963b4836
CW		 2130 if (i915_gem_object_is_purgeable(obj_priv))
2131 i915_gem_object_truncate(obj);
2132
1c5d22f7
CW		 2133 trace_i915_gem_object_unbind(obj);
2134
8dc1775d 2135 return ret;
673a394b
EA		 2136}
2137
b47eb4a2 2138int
4df2faf4
DV		 2139i915_gpu_idle(struct drm_device *dev)
2140{
2141 drm_i915_private_t *dev_priv = dev->dev_private;
2142 bool lists_empty;
c78ec30b 2143 u32 seqno;
852835f3 2144 int ret;
4df2faf4 2145
d1b851fc
ZN		 2146 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2147 list_empty(&dev_priv->render_ring.active_list) &&
2148 (!HAS_BSD(dev) ||
2149 list_empty(&dev_priv->bsd_ring.active_list)));
4df2faf4
DV		 2150 if (lists_empty)
2151 return 0;
2152
2153 /* Flush everything onto the inactive list. */
5c12a07e 2154 seqno = dev_priv->next_seqno;
c78ec30b 2155 i915_gem_flush_ring(dev, NULL, &dev_priv->render_ring,
9220434a 2156 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
c78ec30b 2157 ret = i915_wait_request(dev, seqno, &dev_priv->render_ring);
8a1a49f9
DV		 2158 if (ret)
2159 return ret;
d1b851fc
ZN		 2160
2161 if (HAS_BSD(dev)) {
5c12a07e 2162 seqno = dev_priv->next_seqno;
c78ec30b 2163 i915_gem_flush_ring(dev, NULL, &dev_priv->bsd_ring,
9220434a 2164 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
c78ec30b 2165 ret = i915_wait_request(dev, seqno, &dev_priv->bsd_ring);
d1b851fc
ZN		 2166 if (ret)
2167 return ret;
2168 }
2169
8a1a49f9 2170 return 0;
4df2faf4
DV		 2171}
2172
6911a9b8 2173int
4bdadb97
CW		 2174i915_gem_object_get_pages(struct drm_gem_object *obj,
2175 gfp_t gfpmask)
673a394b 2176{
23010e43 2177 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2178 int page_count, i;
2179 struct address_space *mapping;
2180 struct inode *inode;
2181 struct page *page;
673a394b 2182
778c3544
DV		 2183 BUG_ON(obj_priv->pages_refcount
2184 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2185
856fa198 2186 if (obj_priv->pages_refcount++ != 0)
673a394b
EA		 2187 return 0;
2188
2189 /* Get the list of pages out of our struct file. They'll be pinned
2190 * at this point until we release them.
2191 */
2192 page_count = obj->size / PAGE_SIZE;
856fa198 2193 BUG_ON(obj_priv->pages != NULL);
8e7d2b2c 2194 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
856fa198 2195 if (obj_priv->pages == NULL) {
856fa198 2196 obj_priv->pages_refcount--;
673a394b
EA		 2197 return -ENOMEM;
2198 }
2199
2200 inode = obj->filp->f_path.dentry->d_inode;
2201 mapping = inode->i_mapping;
2202 for (i = 0; i < page_count; i++) {
4bdadb97 2203 page = read_cache_page_gfp(mapping, i,
985b823b 2204 GFP_HIGHUSER |
4bdadb97 2205 __GFP_COLD |
cd9f040d 2206 __GFP_RECLAIMABLE |
4bdadb97 2207 gfpmask);
1f2b1013
CW		 2208 if (IS_ERR(page))
2209 goto err_pages;
2210
856fa198 2211 obj_priv->pages[i] = page;
673a394b 2212 }
280b713b
EA		 2213
2214 if (obj_priv->tiling_mode != I915_TILING_NONE)
2215 i915_gem_object_do_bit_17_swizzle(obj);
2216
673a394b 2217 return 0;
1f2b1013
CW		 2218
2219err_pages:
2220 while (i--)
2221 page_cache_release(obj_priv->pages[i]);
2222
2223 drm_free_large(obj_priv->pages);
2224 obj_priv->pages = NULL;
2225 obj_priv->pages_refcount--;
2226 return PTR_ERR(page);
673a394b
EA		 2227}
2228
4e901fdc
EA		 2229static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2230{
2231 struct drm_gem_object *obj = reg->obj;
2232 struct drm_device *dev = obj->dev;
2233 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2234 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4e901fdc
EA		 2235 int regnum = obj_priv->fence_reg;
2236 uint64_t val;
2237
2238 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2239 0xfffff000) << 32;
2240 val |= obj_priv->gtt_offset & 0xfffff000;
2241 val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2242 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2243
2244 if (obj_priv->tiling_mode == I915_TILING_Y)
2245 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2246 val |= I965_FENCE_REG_VALID;
2247
2248 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2249}
2250
de151cf6
JB		 2251static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2252{
2253 struct drm_gem_object *obj = reg->obj;
2254 struct drm_device *dev = obj->dev;
2255 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2256 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 2257 int regnum = obj_priv->fence_reg;
2258 uint64_t val;
2259
2260 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2261 0xfffff000) << 32;
2262 val |= obj_priv->gtt_offset & 0xfffff000;
2263 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2264 if (obj_priv->tiling_mode == I915_TILING_Y)
2265 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2266 val |= I965_FENCE_REG_VALID;
2267
2268 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2269}
2270
2271static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2272{
2273 struct drm_gem_object *obj = reg->obj;
2274 struct drm_device *dev = obj->dev;
2275 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2276 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2277 int regnum = obj_priv->fence_reg;
0f973f27 2278 int tile_width;
dc529a4f 2279 uint32_t fence_reg, val;
de151cf6
JB		 2280 uint32_t pitch_val;
2281
2282 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2283 (obj_priv->gtt_offset & (obj->size - 1))) {
f06da264 2284 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
0f973f27 2285 __func__, obj_priv->gtt_offset, obj->size);
de151cf6
JB		 2286 return;
2287 }
2288
0f973f27
JB		 2289 if (obj_priv->tiling_mode == I915_TILING_Y &&
2290 HAS_128_BYTE_Y_TILING(dev))
2291 tile_width = 128;
de151cf6 2292 else
0f973f27
JB		 2293 tile_width = 512;
2294
2295 /* Note: pitch better be a power of two tile widths */
2296 pitch_val = obj_priv->stride / tile_width;
2297 pitch_val = ffs(pitch_val) - 1;
de151cf6 2298
c36a2a6d
DV		 2299 if (obj_priv->tiling_mode == I915_TILING_Y &&
2300 HAS_128_BYTE_Y_TILING(dev))
2301 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2302 else
2303 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2304
de151cf6
JB		 2305 val = obj_priv->gtt_offset;
2306 if (obj_priv->tiling_mode == I915_TILING_Y)
2307 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2308 val |= I915_FENCE_SIZE_BITS(obj->size);
2309 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2310 val |= I830_FENCE_REG_VALID;
2311
dc529a4f
EA		 2312 if (regnum < 8)
2313 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2314 else
2315 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2316 I915_WRITE(fence_reg, val);
de151cf6
JB		 2317}
2318
2319static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2320{
2321 struct drm_gem_object *obj = reg->obj;
2322 struct drm_device *dev = obj->dev;
2323 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2324 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 2325 int regnum = obj_priv->fence_reg;
2326 uint32_t val;
2327 uint32_t pitch_val;
8d7773a3 2328 uint32_t fence_size_bits;
de151cf6 2329
8d7773a3 2330 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
de151cf6 2331 (obj_priv->gtt_offset & (obj->size - 1))) {
8d7773a3 2332 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
0f973f27 2333 __func__, obj_priv->gtt_offset);
de151cf6
JB		 2334 return;
2335 }
2336
e76a16de
EA		 2337 pitch_val = obj_priv->stride / 128;
2338 pitch_val = ffs(pitch_val) - 1;
2339 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2340
de151cf6
JB		 2341 val = obj_priv->gtt_offset;
2342 if (obj_priv->tiling_mode == I915_TILING_Y)
2343 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
8d7773a3
DV		 2344 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2345 WARN_ON(fence_size_bits & ~0x00000f00);
2346 val |= fence_size_bits;
de151cf6
JB		 2347 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2348 val |= I830_FENCE_REG_VALID;
2349
2350 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
de151cf6
JB		 2351}
2352
2cf34d7b
CW		 2353static int i915_find_fence_reg(struct drm_device *dev,
2354 bool interruptible)
ae3db24a
DV		 2355{
2356 struct drm_i915_fence_reg *reg = NULL;
2357 struct drm_i915_gem_object *obj_priv = NULL;
2358 struct drm_i915_private *dev_priv = dev->dev_private;
2359 struct drm_gem_object *obj = NULL;
2360 int i, avail, ret;
2361
2362 /* First try to find a free reg */
2363 avail = 0;
2364 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2365 reg = &dev_priv->fence_regs[i];
2366 if (!reg->obj)
2367 return i;
2368
23010e43 2369 obj_priv = to_intel_bo(reg->obj);
ae3db24a
DV		 2370 if (!obj_priv->pin_count)
2371 avail++;
2372 }
2373
2374 if (avail == 0)
2375 return -ENOSPC;
2376
2377 /* None available, try to steal one or wait for a user to finish */
2378 i = I915_FENCE_REG_NONE;
007cc8ac
DV		 2379 list_for_each_entry(reg, &dev_priv->mm.fence_list,
2380 lru_list) {
2381 obj = reg->obj;
2382 obj_priv = to_intel_bo(obj);
ae3db24a
DV		 2383
2384 if (obj_priv->pin_count)
2385 continue;
2386
2387 /* found one! */
2388 i = obj_priv->fence_reg;
2389 break;
2390 }
2391
2392 BUG_ON(i == I915_FENCE_REG_NONE);
2393
2394 /* We only have a reference on obj from the active list. put_fence_reg
2395 * might drop that one, causing a use-after-free in it. So hold a
2396 * private reference to obj like the other callers of put_fence_reg
2397 * (set_tiling ioctl) do. */
2398 drm_gem_object_reference(obj);
2cf34d7b 2399 ret = i915_gem_object_put_fence_reg(obj, interruptible);
ae3db24a
DV		 2400 drm_gem_object_unreference(obj);
2401 if (ret != 0)
2402 return ret;
2403
2404 return i;
2405}
2406
de151cf6
JB		 2407/**
2408 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2409 * @obj: object to map through a fence reg
2410 *
2411 * When mapping objects through the GTT, userspace wants to be able to write
2412 * to them without having to worry about swizzling if the object is tiled.
2413 *
2414 * This function walks the fence regs looking for a free one for @obj,
2415 * stealing one if it can't find any.
2416 *
2417 * It then sets up the reg based on the object's properties: address, pitch
2418 * and tiling format.
2419 */
8c4b8c3f 2420int
2cf34d7b
CW		 2421i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2422 bool interruptible)
de151cf6
JB		 2423{
2424 struct drm_device *dev = obj->dev;
79e53945 2425 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2426 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2427 struct drm_i915_fence_reg *reg = NULL;
ae3db24a 2428 int ret;
de151cf6 2429
a09ba7fa
EA		 2430 /* Just update our place in the LRU if our fence is getting used. */
2431 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
007cc8ac
DV		 2432 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2433 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
a09ba7fa
EA		 2434 return 0;
2435 }
2436
de151cf6
JB		 2437 switch (obj_priv->tiling_mode) {
2438 case I915_TILING_NONE:
2439 WARN(1, "allocating a fence for non-tiled object?\n");
2440 break;
2441 case I915_TILING_X:
0f973f27
JB		 2442 if (!obj_priv->stride)
2443 return -EINVAL;
2444 WARN((obj_priv->stride & (512 - 1)),
2445 "object 0x%08x is X tiled but has non-512B pitch\n",
2446 obj_priv->gtt_offset);
de151cf6
JB		 2447 break;
2448 case I915_TILING_Y:
0f973f27
JB		 2449 if (!obj_priv->stride)
2450 return -EINVAL;
2451 WARN((obj_priv->stride & (128 - 1)),
2452 "object 0x%08x is Y tiled but has non-128B pitch\n",
2453 obj_priv->gtt_offset);
de151cf6
JB		 2454 break;
2455 }
2456
2cf34d7b 2457 ret = i915_find_fence_reg(dev, interruptible);
ae3db24a
DV		 2458 if (ret < 0)
2459 return ret;
de151cf6 2460
ae3db24a
DV		 2461 obj_priv->fence_reg = ret;
2462 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
007cc8ac 2463 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
a09ba7fa 2464
de151cf6
JB		 2465 reg->obj = obj;
2466
e259befd
CW		 2467 switch (INTEL_INFO(dev)->gen) {
2468 case 6:
4e901fdc 2469 sandybridge_write_fence_reg(reg);
e259befd
CW		 2470 break;
2471 case 5:
2472 case 4:
de151cf6 2473 i965_write_fence_reg(reg);
e259befd
CW		 2474 break;
2475 case 3:
de151cf6 2476 i915_write_fence_reg(reg);
e259befd
CW		 2477 break;
2478 case 2:
de151cf6 2479 i830_write_fence_reg(reg);
e259befd
CW		 2480 break;
2481 }
d9ddcb96 2482
ae3db24a
DV		 2483 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2484 obj_priv->tiling_mode);
1c5d22f7 2485
d9ddcb96 2486 return 0;
de151cf6
JB		 2487}
2488
2489/**
2490 * i915_gem_clear_fence_reg - clear out fence register info
2491 * @obj: object to clear
2492 *
2493 * Zeroes out the fence register itself and clears out the associated
2494 * data structures in dev_priv and obj_priv.
2495 */
2496static void
2497i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2498{
2499 struct drm_device *dev = obj->dev;
79e53945 2500 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2501 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
007cc8ac
DV		 2502 struct drm_i915_fence_reg *reg =
2503 &dev_priv->fence_regs[obj_priv->fence_reg];
e259befd 2504 uint32_t fence_reg;
de151cf6 2505
e259befd
CW		 2506 switch (INTEL_INFO(dev)->gen) {
2507 case 6:
4e901fdc
EA		 2508 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2509 (obj_priv->fence_reg * 8), 0);
e259befd
CW		 2510 break;
2511 case 5:
2512 case 4:
de151cf6 2513 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
e259befd
CW		 2514 break;
2515 case 3:
2516 if (obj_priv->fence_reg > 8)
2517 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
dc529a4f 2518 else
e259befd
CW		 2519 case 2:
2520 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
dc529a4f
EA		 2521
2522 I915_WRITE(fence_reg, 0);
e259befd 2523 break;
dc529a4f 2524 }
de151cf6 2525
007cc8ac 2526 reg->obj = NULL;
de151cf6 2527 obj_priv->fence_reg = I915_FENCE_REG_NONE;
007cc8ac 2528 list_del_init(&reg->lru_list);
de151cf6
JB		 2529}
2530
52dc7d32
CW		 2531/**
2532 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2533 * to the buffer to finish, and then resets the fence register.
2534 * @obj: tiled object holding a fence register.
2cf34d7b 2535 * @bool: whether the wait upon the fence is interruptible
52dc7d32
CW		 2536 *
2537 * Zeroes out the fence register itself and clears out the associated
2538 * data structures in dev_priv and obj_priv.
2539 */
2540int
2cf34d7b
CW		 2541i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2542 bool interruptible)
52dc7d32
CW		 2543{
2544 struct drm_device *dev = obj->dev;
53640e1d 2545 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2546 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
53640e1d 2547 struct drm_i915_fence_reg *reg;
52dc7d32
CW		 2548
2549 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2550 return 0;
2551
10ae9bd2
DV		 2552 /* If we've changed tiling, GTT-mappings of the object
2553 * need to re-fault to ensure that the correct fence register
2554 * setup is in place.
2555 */
2556 i915_gem_release_mmap(obj);
2557
52dc7d32
CW		 2558 /* On the i915, GPU access to tiled buffers is via a fence,
2559 * therefore we must wait for any outstanding access to complete
2560 * before clearing the fence.
2561 */
53640e1d
CW		 2562 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2563 if (reg->gpu) {
52dc7d32
CW		 2564 int ret;
2565
2cf34d7b 2566 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
0bc23aad
CW		 2567 if (ret)
2568 return ret;
2569
2cf34d7b 2570 ret = i915_gem_object_wait_rendering(obj, interruptible);
0bc23aad 2571 if (ret)
52dc7d32 2572 return ret;
53640e1d
CW		 2573
2574 reg->gpu = false;
52dc7d32
CW		 2575 }
2576
4a726612 2577 i915_gem_object_flush_gtt_write_domain(obj);
0bc23aad 2578 i915_gem_clear_fence_reg(obj);
52dc7d32
CW		 2579
2580 return 0;
2581}
2582
673a394b
EA		 2583/**
2584 * Finds free space in the GTT aperture and binds the object there.
2585 */
2586static int
2587i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2588{
2589 struct drm_device *dev = obj->dev;
2590 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2591 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 2592 struct drm_mm_node *free_space;
4bdadb97 2593 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
07f73f69 2594 int ret;
673a394b 2595
bb6baf76 2596 if (obj_priv->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2597 DRM_ERROR("Attempting to bind a purgeable object\n");
2598 return -EINVAL;
2599 }
2600
673a394b 2601 if (alignment == 0)
0f973f27 2602 alignment = i915_gem_get_gtt_alignment(obj);
8d7773a3 2603 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
673a394b
EA		 2604 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2605 return -EINVAL;
2606 }
2607
654fc607
CW		 2608 /* If the object is bigger than the entire aperture, reject it early
2609 * before evicting everything in a vain attempt to find space.
2610 */
2611 if (obj->size > dev->gtt_total) {
2612 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2613 return -E2BIG;
2614 }
2615
673a394b
EA		 2616 search_free:
2617 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2618 obj->size, alignment, 0);
2619 if (free_space != NULL) {
2620 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2621 alignment);
db3307a9 2622 if (obj_priv->gtt_space != NULL)
673a394b 2623 obj_priv->gtt_offset = obj_priv->gtt_space->start;
673a394b
EA		 2624 }
2625 if (obj_priv->gtt_space == NULL) {
2626 /* If the gtt is empty and we're still having trouble
2627 * fitting our object in, we're out of memory.
2628 */
2629#if WATCH_LRU
2630 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2631#endif
0108a3ed 2632 ret = i915_gem_evict_something(dev, obj->size, alignment);
9731129c 2633 if (ret)
673a394b 2634 return ret;
9731129c 2635
673a394b
EA		 2636 goto search_free;
2637 }
2638
2639#if WATCH_BUF
cfd43c02 2640 DRM_INFO("Binding object of size %zd at 0x%08x\n",
673a394b
EA		 2641 obj->size, obj_priv->gtt_offset);
2642#endif
4bdadb97 2643 ret = i915_gem_object_get_pages(obj, gfpmask);
673a394b
EA		 2644 if (ret) {
2645 drm_mm_put_block(obj_priv->gtt_space);
2646 obj_priv->gtt_space = NULL;
07f73f69
CW		 2647
2648 if (ret == -ENOMEM) {
2649 /* first try to clear up some space from the GTT */
0108a3ed
DV		 2650 ret = i915_gem_evict_something(dev, obj->size,
2651 alignment);
07f73f69 2652 if (ret) {
07f73f69 2653 /* now try to shrink everyone else */
4bdadb97
CW		 2654 if (gfpmask) {
2655 gfpmask = 0;
2656 goto search_free;
07f73f69
CW		 2657 }
2658
2659 return ret;
2660 }
2661
2662 goto search_free;
2663 }
2664
673a394b
EA		 2665 return ret;
2666 }
2667
673a394b
EA		 2668 /* Create an AGP memory structure pointing at our pages, and bind it
2669 * into the GTT.
2670 */
2671 obj_priv->agp_mem = drm_agp_bind_pages(dev,
856fa198 2672 obj_priv->pages,
07f73f69 2673 obj->size >> PAGE_SHIFT,
ba1eb1d8
KP		 2674 obj_priv->gtt_offset,
2675 obj_priv->agp_type);
673a394b 2676 if (obj_priv->agp_mem == NULL) {
856fa198 2677 i915_gem_object_put_pages(obj);
673a394b
EA		 2678 drm_mm_put_block(obj_priv->gtt_space);
2679 obj_priv->gtt_space = NULL;
07f73f69 2680
0108a3ed 2681 ret = i915_gem_evict_something(dev, obj->size, alignment);
9731129c 2682 if (ret)
07f73f69 2683 return ret;
07f73f69
CW		 2684
2685 goto search_free;
673a394b
EA		 2686 }
2687 atomic_inc(&dev->gtt_count);
2688 atomic_add(obj->size, &dev->gtt_memory);
2689
bf1a1092
CW		 2690 /* keep track of bounds object by adding it to the inactive list */
2691 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2692
673a394b
EA		 2693 /* Assert that the object is not currently in any GPU domain. As it
2694 * wasn't in the GTT, there shouldn't be any way it could have been in
2695 * a GPU cache
2696 */
21d509e3
CW		 2697 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2698 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2699
1c5d22f7
CW		 2700 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2701
673a394b
EA		 2702 return 0;
2703}
2704
2705void
2706i915_gem_clflush_object(struct drm_gem_object *obj)
2707{
23010e43 2708 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2709
2710 /* If we don't have a page list set up, then we're not pinned
2711 * to GPU, and we can ignore the cache flush because it'll happen
2712 * again at bind time.
2713 */
856fa198 2714 if (obj_priv->pages == NULL)
673a394b
EA		 2715 return;
2716
1c5d22f7 2717 trace_i915_gem_object_clflush(obj);
cfa16a0d 2718
856fa198 2719 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
673a394b
EA		 2720}
2721
e47c68e9 2722/** Flushes any GPU write domain for the object if it's dirty. */
2dafb1e0 2723static int
ba3d8d74
DV		 2724i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2725 bool pipelined)
e47c68e9
EA		 2726{
2727 struct drm_device *dev = obj->dev;
1c5d22f7 2728 uint32_t old_write_domain;
e47c68e9
EA		 2729
2730 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2dafb1e0 2731 return 0;
e47c68e9
EA		 2732
2733 /* Queue the GPU write cache flushing we need. */
1c5d22f7 2734 old_write_domain = obj->write_domain;
c78ec30b 2735 i915_gem_flush_ring(dev, NULL,
9220434a
CW		 2736 to_intel_bo(obj)->ring,
2737 0, obj->write_domain);
48b956c5 2738 BUG_ON(obj->write_domain);
1c5d22f7
CW		 2739
2740 trace_i915_gem_object_change_domain(obj,
2741 obj->read_domains,
2742 old_write_domain);
ba3d8d74
DV		 2743
2744 if (pipelined)
2745 return 0;
2746
2cf34d7b 2747 return i915_gem_object_wait_rendering(obj, true);
e47c68e9
EA		 2748}
2749
2750/** Flushes the GTT write domain for the object if it's dirty. */
2751static void
2752i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2753{
1c5d22f7
CW		 2754 uint32_t old_write_domain;
2755
e47c68e9
EA		 2756 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2757 return;
2758
2759 /* No actual flushing is required for the GTT write domain. Writes
2760 * to it immediately go to main memory as far as we know, so there's
2761 * no chipset flush. It also doesn't land in render cache.
2762 */
1c5d22f7 2763 old_write_domain = obj->write_domain;
e47c68e9 2764 obj->write_domain = 0;
1c5d22f7
CW		 2765
2766 trace_i915_gem_object_change_domain(obj,
2767 obj->read_domains,
2768 old_write_domain);
e47c68e9
EA		 2769}
2770
2771/** Flushes the CPU write domain for the object if it's dirty. */
2772static void
2773i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2774{
2775 struct drm_device *dev = obj->dev;
1c5d22f7 2776 uint32_t old_write_domain;
e47c68e9
EA		 2777
2778 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2779 return;
2780
2781 i915_gem_clflush_object(obj);
2782 drm_agp_chipset_flush(dev);
1c5d22f7 2783 old_write_domain = obj->write_domain;
e47c68e9 2784 obj->write_domain = 0;
1c5d22f7
CW		 2785
2786 trace_i915_gem_object_change_domain(obj,
2787 obj->read_domains,
2788 old_write_domain);
e47c68e9
EA		 2789}
2790
2ef7eeaa
EA		 2791/**
2792 * Moves a single object to the GTT read, and possibly write domain.
2793 *
2794 * This function returns when the move is complete, including waiting on
2795 * flushes to occur.
2796 */
79e53945 2797int
2ef7eeaa
EA		 2798i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2799{
23010e43 2800 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 2801 uint32_t old_write_domain, old_read_domains;
e47c68e9 2802 int ret;
2ef7eeaa 2803
02354392
EA		 2804 /* Not valid to be called on unbound objects. */
2805 if (obj_priv->gtt_space == NULL)
2806 return -EINVAL;
2807
ba3d8d74 2808 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9
EA		 2809 if (ret != 0)
2810 return ret;
2811
7213342d 2812 i915_gem_object_flush_cpu_write_domain(obj);
1c5d22f7 2813
ba3d8d74 2814 if (write) {
2cf34d7b 2815 ret = i915_gem_object_wait_rendering(obj, true);
ba3d8d74
DV		 2816 if (ret)
2817 return ret;
ba3d8d74 2818 }
2ef7eeaa 2819
7213342d
CW		 2820 old_write_domain = obj->write_domain;
2821 old_read_domains = obj->read_domains;
2ef7eeaa 2822
e47c68e9
EA		 2823 /* It should now be out of any other write domains, and we can update
2824 * the domain values for our changes.
2825 */
2826 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2827 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2828 if (write) {
7213342d 2829 obj->read_domains = I915_GEM_DOMAIN_GTT;
e47c68e9
EA		 2830 obj->write_domain = I915_GEM_DOMAIN_GTT;
2831 obj_priv->dirty = 1;
2ef7eeaa
EA		 2832 }
2833
1c5d22f7
CW		 2834 trace_i915_gem_object_change_domain(obj,
2835 old_read_domains,
2836 old_write_domain);
2837
e47c68e9
EA		 2838 return 0;
2839}
2840
b9241ea3
ZW		 2841/*
2842 * Prepare buffer for display plane. Use uninterruptible for possible flush
2843 * wait, as in modesetting process we're not supposed to be interrupted.
2844 */
2845int
48b956c5
CW		 2846i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2847 bool pipelined)
b9241ea3 2848{
23010e43 2849 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ba3d8d74 2850 uint32_t old_read_domains;
b9241ea3
ZW		 2851 int ret;
2852
2853 /* Not valid to be called on unbound objects. */
2854 if (obj_priv->gtt_space == NULL)
2855 return -EINVAL;
2856
48b956c5
CW		 2857 ret = i915_gem_object_flush_gpu_write_domain(obj, pipelined);
2858 if (ret)
e35a41de 2859 return ret;
b9241ea3 2860
b118c1e3
CW		 2861 i915_gem_object_flush_cpu_write_domain(obj);
2862
b9241ea3 2863 old_read_domains = obj->read_domains;
c78ec30b 2864 obj->read_domains |= I915_GEM_DOMAIN_GTT;
b9241ea3
ZW		 2865
2866 trace_i915_gem_object_change_domain(obj,
2867 old_read_domains,
ba3d8d74 2868 obj->write_domain);
b9241ea3
ZW		 2869
2870 return 0;
2871}
2872
e47c68e9
EA		 2873/**
2874 * Moves a single object to the CPU read, and possibly write domain.
2875 *
2876 * This function returns when the move is complete, including waiting on
2877 * flushes to occur.
2878 */
2879static int
2880i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2881{
1c5d22f7 2882 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 2883 int ret;
2884
ba3d8d74 2885 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9
EA		 2886 if (ret != 0)
2887 return ret;
2ef7eeaa 2888
e47c68e9 2889 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 2890
e47c68e9
EA		 2891 /* If we have a partially-valid cache of the object in the CPU,
2892 * finish invalidating it and free the per-page flags.
2ef7eeaa 2893 */
e47c68e9 2894 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 2895
7213342d 2896 if (write) {
2cf34d7b 2897 ret = i915_gem_object_wait_rendering(obj, true);
7213342d
CW		 2898 if (ret)
2899 return ret;
2900 }
2901
1c5d22f7
CW		 2902 old_write_domain = obj->write_domain;
2903 old_read_domains = obj->read_domains;
2904
e47c68e9
EA		 2905 /* Flush the CPU cache if it's still invalid. */
2906 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 2907 i915_gem_clflush_object(obj);
2ef7eeaa 2908
e47c68e9 2909 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 2910 }
2911
2912 /* It should now be out of any other write domains, and we can update
2913 * the domain values for our changes.
2914 */
e47c68e9
EA		 2915 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2916
2917 /* If we're writing through the CPU, then the GPU read domains will
2918 * need to be invalidated at next use.
2919 */
2920 if (write) {
c78ec30b 2921 obj->read_domains = I915_GEM_DOMAIN_CPU;
e47c68e9
EA		 2922 obj->write_domain = I915_GEM_DOMAIN_CPU;
2923 }
2ef7eeaa 2924
1c5d22f7
CW		 2925 trace_i915_gem_object_change_domain(obj,
2926 old_read_domains,
2927 old_write_domain);
2928
2ef7eeaa
EA		 2929 return 0;
2930}
2931
673a394b
EA		 2932/*
2933 * Set the next domain for the specified object. This
2934 * may not actually perform the necessary flushing/invaliding though,
2935 * as that may want to be batched with other set_domain operations
2936 *
2937 * This is (we hope) the only really tricky part of gem. The goal
2938 * is fairly simple -- track which caches hold bits of the object
2939 * and make sure they remain coherent. A few concrete examples may
2940 * help to explain how it works. For shorthand, we use the notation
2941 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2942 * a pair of read and write domain masks.
2943 *
2944 * Case 1: the batch buffer
2945 *
2946 * 1. Allocated
2947 * 2. Written by CPU
2948 * 3. Mapped to GTT
2949 * 4. Read by GPU
2950 * 5. Unmapped from GTT
2951 * 6. Freed
2952 *
2953 * Let's take these a step at a time
2954 *
2955 * 1. Allocated
2956 * Pages allocated from the kernel may still have
2957 * cache contents, so we set them to (CPU, CPU) always.
2958 * 2. Written by CPU (using pwrite)
2959 * The pwrite function calls set_domain (CPU, CPU) and
2960 * this function does nothing (as nothing changes)
2961 * 3. Mapped by GTT
2962 * This function asserts that the object is not
2963 * currently in any GPU-based read or write domains
2964 * 4. Read by GPU
2965 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2966 * As write_domain is zero, this function adds in the
2967 * current read domains (CPU+COMMAND, 0).
2968 * flush_domains is set to CPU.
2969 * invalidate_domains is set to COMMAND
2970 * clflush is run to get data out of the CPU caches
2971 * then i915_dev_set_domain calls i915_gem_flush to
2972 * emit an MI_FLUSH and drm_agp_chipset_flush
2973 * 5. Unmapped from GTT
2974 * i915_gem_object_unbind calls set_domain (CPU, CPU)
2975 * flush_domains and invalidate_domains end up both zero
2976 * so no flushing/invalidating happens
2977 * 6. Freed
2978 * yay, done
2979 *
2980 * Case 2: The shared render buffer
2981 *
2982 * 1. Allocated
2983 * 2. Mapped to GTT
2984 * 3. Read/written by GPU
2985 * 4. set_domain to (CPU,CPU)
2986 * 5. Read/written by CPU
2987 * 6. Read/written by GPU
2988 *
2989 * 1. Allocated
2990 * Same as last example, (CPU, CPU)
2991 * 2. Mapped to GTT
2992 * Nothing changes (assertions find that it is not in the GPU)
2993 * 3. Read/written by GPU
2994 * execbuffer calls set_domain (RENDER, RENDER)
2995 * flush_domains gets CPU
2996 * invalidate_domains gets GPU
2997 * clflush (obj)
2998 * MI_FLUSH and drm_agp_chipset_flush
2999 * 4. set_domain (CPU, CPU)
3000 * flush_domains gets GPU
3001 * invalidate_domains gets CPU
3002 * wait_rendering (obj) to make sure all drawing is complete.
3003 * This will include an MI_FLUSH to get the data from GPU
3004 * to memory
3005 * clflush (obj) to invalidate the CPU cache
3006 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3007 * 5. Read/written by CPU
3008 * cache lines are loaded and dirtied
3009 * 6. Read written by GPU
3010 * Same as last GPU access
3011 *
3012 * Case 3: The constant buffer
3013 *
3014 * 1. Allocated
3015 * 2. Written by CPU
3016 * 3. Read by GPU
3017 * 4. Updated (written) by CPU again
3018 * 5. Read by GPU
3019 *
3020 * 1. Allocated
3021 * (CPU, CPU)
3022 * 2. Written by CPU
3023 * (CPU, CPU)
3024 * 3. Read by GPU
3025 * (CPU+RENDER, 0)
3026 * flush_domains = CPU
3027 * invalidate_domains = RENDER
3028 * clflush (obj)
3029 * MI_FLUSH
3030 * drm_agp_chipset_flush
3031 * 4. Updated (written) by CPU again
3032 * (CPU, CPU)
3033 * flush_domains = 0 (no previous write domain)
3034 * invalidate_domains = 0 (no new read domains)
3035 * 5. Read by GPU
3036 * (CPU+RENDER, 0)
3037 * flush_domains = CPU
3038 * invalidate_domains = RENDER
3039 * clflush (obj)
3040 * MI_FLUSH
3041 * drm_agp_chipset_flush
3042 */
c0d90829 3043static void
8b0e378a 3044i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
673a394b
EA		 3045{
3046 struct drm_device *dev = obj->dev;
9220434a 3047 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 3048 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 3049 uint32_t invalidate_domains = 0;
3050 uint32_t flush_domains = 0;
1c5d22f7 3051 uint32_t old_read_domains;
e47c68e9 3052
8b0e378a
EA		 3053 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3054 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
673a394b 3055
652c393a
JB		 3056 intel_mark_busy(dev, obj);
3057
673a394b
EA		 3058#if WATCH_BUF
3059 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
3060 __func__, obj,
8b0e378a
EA		 3061 obj->read_domains, obj->pending_read_domains,
3062 obj->write_domain, obj->pending_write_domain);
673a394b
EA		 3063#endif
3064 /*
3065 * If the object isn't moving to a new write domain,
3066 * let the object stay in multiple read domains
3067 */
8b0e378a
EA		 3068 if (obj->pending_write_domain == 0)
3069 obj->pending_read_domains |= obj->read_domains;
673a394b
EA		 3070 else
3071 obj_priv->dirty = 1;
3072
3073 /*
3074 * Flush the current write domain if
3075 * the new read domains don't match. Invalidate
3076 * any read domains which differ from the old
3077 * write domain
3078 */
8b0e378a
EA		 3079 if (obj->write_domain &&
3080 obj->write_domain != obj->pending_read_domains) {
673a394b 3081 flush_domains |= obj->write_domain;
8b0e378a
EA		 3082 invalidate_domains |=
3083 obj->pending_read_domains & ~obj->write_domain;
673a394b
EA		 3084 }
3085 /*
3086 * Invalidate any read caches which may have
3087 * stale data. That is, any new read domains.
3088 */
8b0e378a 3089 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
673a394b
EA		 3090 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3091#if WATCH_BUF
3092 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3093 __func__, flush_domains, invalidate_domains);
3094#endif
673a394b
EA		 3095 i915_gem_clflush_object(obj);
3096 }
3097
1c5d22f7
CW		 3098 old_read_domains = obj->read_domains;
3099
efbeed96
EA		 3100 /* The actual obj->write_domain will be updated with
3101 * pending_write_domain after we emit the accumulated flush for all
3102 * of our domain changes in execbuffers (which clears objects'
3103 * write_domains). So if we have a current write domain that we
3104 * aren't changing, set pending_write_domain to that.
3105 */
3106 if (flush_domains == 0 && obj->pending_write_domain == 0)
3107 obj->pending_write_domain = obj->write_domain;
8b0e378a 3108 obj->read_domains = obj->pending_read_domains;
673a394b
EA		 3109
3110 dev->invalidate_domains |= invalidate_domains;
3111 dev->flush_domains |= flush_domains;
9220434a
CW		 3112 if (obj_priv->ring)
3113 dev_priv->mm.flush_rings |= obj_priv->ring->id;
673a394b
EA		 3114#if WATCH_BUF
3115 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3116 __func__,
3117 obj->read_domains, obj->write_domain,
3118 dev->invalidate_domains, dev->flush_domains);
3119#endif
1c5d22f7
CW		 3120
3121 trace_i915_gem_object_change_domain(obj,
3122 old_read_domains,
3123 obj->write_domain);
673a394b
EA		 3124}
3125
3126/**
e47c68e9 3127 * Moves the object from a partially CPU read to a full one.
673a394b 3128 *
e47c68e9
EA		 3129 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3130 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 3131 */
e47c68e9
EA		 3132static void
3133i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
673a394b 3134{
23010e43 3135 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 3136
e47c68e9
EA		 3137 if (!obj_priv->page_cpu_valid)
3138 return;
3139
3140 /* If we're partially in the CPU read domain, finish moving it in.
3141 */
3142 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3143 int i;
3144
3145 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3146 if (obj_priv->page_cpu_valid[i])
3147 continue;
856fa198 3148 drm_clflush_pages(obj_priv->pages + i, 1);
e47c68e9 3149 }
e47c68e9
EA		 3150 }
3151
3152 /* Free the page_cpu_valid mappings which are now stale, whether
3153 * or not we've got I915_GEM_DOMAIN_CPU.
3154 */
9a298b2a 3155 kfree(obj_priv->page_cpu_valid);
e47c68e9
EA		 3156 obj_priv->page_cpu_valid = NULL;
3157}
3158
3159/**
3160 * Set the CPU read domain on a range of the object.
3161 *
3162 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3163 * not entirely valid. The page_cpu_valid member of the object flags which
3164 * pages have been flushed, and will be respected by
3165 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3166 * of the whole object.
3167 *
3168 * This function returns when the move is complete, including waiting on
3169 * flushes to occur.
3170 */
3171static int
3172i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3173 uint64_t offset, uint64_t size)
3174{
23010e43 3175 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 3176 uint32_t old_read_domains;
e47c68e9 3177 int i, ret;
673a394b 3178
e47c68e9
EA		 3179 if (offset == 0 && size == obj->size)
3180 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 3181
ba3d8d74 3182 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9 3183 if (ret != 0)
6a47baa6 3184 return ret;
e47c68e9
EA		 3185 i915_gem_object_flush_gtt_write_domain(obj);
3186
3187 /* If we're already fully in the CPU read domain, we're done. */
3188 if (obj_priv->page_cpu_valid == NULL &&
3189 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3190 return 0;
673a394b 3191
e47c68e9
EA		 3192 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3193 * newly adding I915_GEM_DOMAIN_CPU
3194 */
673a394b 3195 if (obj_priv->page_cpu_valid == NULL) {
9a298b2a
EA		 3196 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3197 GFP_KERNEL);
e47c68e9
EA		 3198 if (obj_priv->page_cpu_valid == NULL)
3199 return -ENOMEM;
3200 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3201 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
673a394b
EA		 3202
3203 /* Flush the cache on any pages that are still invalid from the CPU's
3204 * perspective.
3205 */
e47c68e9
EA		 3206 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3207 i++) {
673a394b
EA		 3208 if (obj_priv->page_cpu_valid[i])
3209 continue;
3210
856fa198 3211 drm_clflush_pages(obj_priv->pages + i, 1);
673a394b
EA		 3212
3213 obj_priv->page_cpu_valid[i] = 1;
3214 }
3215
e47c68e9
EA		 3216 /* It should now be out of any other write domains, and we can update
3217 * the domain values for our changes.
3218 */
3219 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3220
1c5d22f7 3221 old_read_domains = obj->read_domains;
e47c68e9
EA		 3222 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3223
1c5d22f7
CW		 3224 trace_i915_gem_object_change_domain(obj,
3225 old_read_domains,
3226 obj->write_domain);
3227
673a394b
EA		 3228 return 0;
3229}
3230
673a394b
EA		 3231/**
3232 * Pin an object to the GTT and evaluate the relocations landing in it.
3233 */
3234static int
3235i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3236 struct drm_file *file_priv,
76446cac 3237 struct drm_i915_gem_exec_object2 *entry,
40a5f0de 3238 struct drm_i915_gem_relocation_entry *relocs)
673a394b
EA		 3239{
3240 struct drm_device *dev = obj->dev;
0839ccb8 3241 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 3242 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 3243 int i, ret;
0839ccb8 3244 void __iomem *reloc_page;
76446cac
JB		 3245 bool need_fence;
3246
3247 need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3248 obj_priv->tiling_mode != I915_TILING_NONE;
3249
3250 /* Check fence reg constraints and rebind if necessary */
808b24d6
CW		 3251 if (need_fence &&
3252 !i915_gem_object_fence_offset_ok(obj,
3253 obj_priv->tiling_mode)) {
3254 ret = i915_gem_object_unbind(obj);
3255 if (ret)
3256 return ret;
3257 }
673a394b
EA		 3258
3259 /* Choose the GTT offset for our buffer and put it there. */
3260 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3261 if (ret)
3262 return ret;
3263
76446cac
JB		 3264 /*
3265 * Pre-965 chips need a fence register set up in order to
3266 * properly handle blits to/from tiled surfaces.
3267 */
3268 if (need_fence) {
53640e1d 3269 ret = i915_gem_object_get_fence_reg(obj, true);
76446cac 3270 if (ret != 0) {
76446cac
JB		 3271 i915_gem_object_unpin(obj);
3272 return ret;
3273 }
53640e1d
CW		 3274
3275 dev_priv->fence_regs[obj_priv->fence_reg].gpu = true;
76446cac
JB		 3276 }
3277
673a394b
EA		 3278 entry->offset = obj_priv->gtt_offset;
3279
673a394b
EA		 3280 /* Apply the relocations, using the GTT aperture to avoid cache
3281 * flushing requirements.
3282 */
3283 for (i = 0; i < entry->relocation_count; i++) {
40a5f0de 3284 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
673a394b
EA		 3285 struct drm_gem_object *target_obj;
3286 struct drm_i915_gem_object *target_obj_priv;
3043c60c
EA		 3287 uint32_t reloc_val, reloc_offset;
3288 uint32_t __iomem *reloc_entry;
673a394b 3289
673a394b 3290 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
40a5f0de 3291 reloc->target_handle);
673a394b
EA		 3292 if (target_obj == NULL) {
3293 i915_gem_object_unpin(obj);
bf79cb91 3294 return -ENOENT;
673a394b 3295 }
23010e43 3296 target_obj_priv = to_intel_bo(target_obj);
673a394b 3297
8542a0bb
CW		 3298#if WATCH_RELOC
3299 DRM_INFO("%s: obj %p offset %08x target %d "
3300 "read %08x write %08x gtt %08x "
3301 "presumed %08x delta %08x\n",
3302 __func__,
3303 obj,
3304 (int) reloc->offset,
3305 (int) reloc->target_handle,
3306 (int) reloc->read_domains,
3307 (int) reloc->write_domain,
3308 (int) target_obj_priv->gtt_offset,
3309 (int) reloc->presumed_offset,
3310 reloc->delta);
3311#endif
3312
673a394b
EA		 3313 /* The target buffer should have appeared before us in the
3314 * exec_object list, so it should have a GTT space bound by now.
3315 */
3316 if (target_obj_priv->gtt_space == NULL) {
3317 DRM_ERROR("No GTT space found for object %d\n",
40a5f0de 3318 reloc->target_handle);
673a394b
EA		 3319 drm_gem_object_unreference(target_obj);
3320 i915_gem_object_unpin(obj);
3321 return -EINVAL;
3322 }
3323
8542a0bb 3324 /* Validate that the target is in a valid r/w GPU domain */
16edd550
DV		 3325 if (reloc->write_domain & (reloc->write_domain - 1)) {
3326 DRM_ERROR("reloc with multiple write domains: "
3327 "obj %p target %d offset %d "
3328 "read %08x write %08x",
3329 obj, reloc->target_handle,
3330 (int) reloc->offset,
3331 reloc->read_domains,
3332 reloc->write_domain);
3333 return -EINVAL;
3334 }
40a5f0de
EA		 3335 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3336 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 3337 DRM_ERROR("reloc with read/write CPU domains: "
3338 "obj %p target %d offset %d "
3339 "read %08x write %08x",
40a5f0de
EA		 3340 obj, reloc->target_handle,
3341 (int) reloc->offset,
3342 reloc->read_domains,
3343 reloc->write_domain);
491152b8
CW		 3344 drm_gem_object_unreference(target_obj);
3345 i915_gem_object_unpin(obj);
e47c68e9
EA		 3346 return -EINVAL;
3347 }
40a5f0de
EA		 3348 if (reloc->write_domain && target_obj->pending_write_domain &&
3349 reloc->write_domain != target_obj->pending_write_domain) {
673a394b
EA		 3350 DRM_ERROR("Write domain conflict: "
3351 "obj %p target %d offset %d "
3352 "new %08x old %08x\n",
40a5f0de
EA		 3353 obj, reloc->target_handle,
3354 (int) reloc->offset,
3355 reloc->write_domain,
673a394b
EA		 3356 target_obj->pending_write_domain);
3357 drm_gem_object_unreference(target_obj);
3358 i915_gem_object_unpin(obj);
3359 return -EINVAL;
3360 }
3361
40a5f0de
EA		 3362 target_obj->pending_read_domains |= reloc->read_domains;
3363 target_obj->pending_write_domain |= reloc->write_domain;
673a394b
EA		 3364
3365 /* If the relocation already has the right value in it, no
3366 * more work needs to be done.
3367 */
40a5f0de 3368 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
673a394b
EA		 3369 drm_gem_object_unreference(target_obj);
3370 continue;
3371 }
3372
8542a0bb
CW		 3373 /* Check that the relocation address is valid... */
3374 if (reloc->offset > obj->size - 4) {
3375 DRM_ERROR("Relocation beyond object bounds: "
3376 "obj %p target %d offset %d size %d.\n",
3377 obj, reloc->target_handle,
3378 (int) reloc->offset, (int) obj->size);
3379 drm_gem_object_unreference(target_obj);
3380 i915_gem_object_unpin(obj);
3381 return -EINVAL;
3382 }
3383 if (reloc->offset & 3) {
3384 DRM_ERROR("Relocation not 4-byte aligned: "
3385 "obj %p target %d offset %d.\n",
3386 obj, reloc->target_handle,
3387 (int) reloc->offset);
3388 drm_gem_object_unreference(target_obj);
3389 i915_gem_object_unpin(obj);
3390 return -EINVAL;
3391 }
3392
3393 /* and points to somewhere within the target object. */
3394 if (reloc->delta >= target_obj->size) {
3395 DRM_ERROR("Relocation beyond target object bounds: "
3396 "obj %p target %d delta %d size %d.\n",
3397 obj, reloc->target_handle,
3398 (int) reloc->delta, (int) target_obj->size);
3399 drm_gem_object_unreference(target_obj);
3400 i915_gem_object_unpin(obj);
3401 return -EINVAL;
3402 }
3403
2ef7eeaa
EA		 3404 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3405 if (ret != 0) {
3406 drm_gem_object_unreference(target_obj);
3407 i915_gem_object_unpin(obj);
3408 return -EINVAL;
673a394b
EA		 3409 }
3410
3411 /* Map the page containing the relocation we're going to
3412 * perform.
3413 */
40a5f0de 3414 reloc_offset = obj_priv->gtt_offset + reloc->offset;
0839ccb8
KP		 3415 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3416 (reloc_offset &
fca3ec01
CW		 3417 ~(PAGE_SIZE - 1)),
3418 KM_USER0);
3043c60c 3419 reloc_entry = (uint32_t __iomem *)(reloc_page +
0839ccb8 3420 (reloc_offset & (PAGE_SIZE - 1)));
40a5f0de 3421 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
673a394b
EA		 3422
3423#if WATCH_BUF
3424 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
40a5f0de 3425 obj, (unsigned int) reloc->offset,
673a394b
EA		 3426 readl(reloc_entry), reloc_val);
3427#endif
3428 writel(reloc_val, reloc_entry);
fca3ec01 3429 io_mapping_unmap_atomic(reloc_page, KM_USER0);
673a394b 3430
40a5f0de
EA		 3431 /* The updated presumed offset for this entry will be
3432 * copied back out to the user.
673a394b 3433 */
40a5f0de 3434 reloc->presumed_offset = target_obj_priv->gtt_offset;
673a394b
EA		 3435
3436 drm_gem_object_unreference(target_obj);
3437 }
3438
673a394b
EA		 3439#if WATCH_BUF
3440 if (0)
3441 i915_gem_dump_object(obj, 128, __func__, ~0);
3442#endif
3443 return 0;
3444}
3445
673a394b
EA		 3446/* Throttle our rendering by waiting until the ring has completed our requests
3447 * emitted over 20 msec ago.
3448 *
b962442e
EA		 3449 * Note that if we were to use the current jiffies each time around the loop,
3450 * we wouldn't escape the function with any frames outstanding if the time to
3451 * render a frame was over 20ms.
3452 *
673a394b
EA		 3453 * This should get us reasonable parallelism between CPU and GPU but also
3454 * relatively low latency when blocking on a particular request to finish.
3455 */
3456static int
f787a5f5 3457i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
673a394b 3458{
f787a5f5
CW		 3459 struct drm_i915_private *dev_priv = dev->dev_private;
3460 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e 3461 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
f787a5f5
CW		 3462 struct drm_i915_gem_request *request;
3463 struct intel_ring_buffer *ring = NULL;
3464 u32 seqno = 0;
3465 int ret;
673a394b 3466
f787a5f5
CW		 3467 mutex_lock(&file_priv->mutex);
3468 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
b962442e
EA		 3469 if (time_after_eq(request->emitted_jiffies, recent_enough))
3470 break;
3471
f787a5f5
CW		 3472 ring = request->ring;
3473 seqno = request->seqno;
b962442e 3474 }
f787a5f5
CW		 3475 mutex_unlock(&file_priv->mutex);
3476
3477 if (seqno == 0)
3478 return 0;
3479
3480 ret = 0;
3481 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
3482 /* And wait for the seqno passing without holding any locks and
3483 * causing extra latency for others. This is safe as the irq
3484 * generation is designed to be run atomically and so is
3485 * lockless.
3486 */
3487 ring->user_irq_get(dev, ring);
3488 ret = wait_event_interruptible(ring->irq_queue,
3489 i915_seqno_passed(ring->get_seqno(dev, ring), seqno)
3490 || atomic_read(&dev_priv->mm.wedged));
3491 ring->user_irq_put(dev, ring);
3492
3493 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3494 ret = -EIO;
3495 }
3496
3497 if (ret == 0)
3498 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
b962442e 3499
673a394b
EA		 3500 return ret;
3501}
3502
40a5f0de 3503static int
76446cac 3504i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
40a5f0de
EA		 3505 uint32_t buffer_count,
3506 struct drm_i915_gem_relocation_entry **relocs)
3507{
3508 uint32_t reloc_count = 0, reloc_index = 0, i;
3509 int ret;
3510
3511 *relocs = NULL;
3512 for (i = 0; i < buffer_count; i++) {
3513 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3514 return -EINVAL;
3515 reloc_count += exec_list[i].relocation_count;
3516 }
3517
8e7d2b2c 3518 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
76446cac
JB		 3519 if (*relocs == NULL) {
3520 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
40a5f0de 3521 return -ENOMEM;
76446cac 3522 }
40a5f0de
EA		 3523
3524 for (i = 0; i < buffer_count; i++) {
3525 struct drm_i915_gem_relocation_entry __user *user_relocs;
3526
3527 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3528
3529 ret = copy_from_user(&(*relocs)[reloc_index],
3530 user_relocs,
3531 exec_list[i].relocation_count *
3532 sizeof(**relocs));
3533 if (ret != 0) {
8e7d2b2c 3534 drm_free_large(*relocs);
40a5f0de 3535 *relocs = NULL;
2bc43b5c 3536 return -EFAULT;
40a5f0de
EA		 3537 }
3538
3539 reloc_index += exec_list[i].relocation_count;
3540 }
3541
2bc43b5c 3542 return 0;
40a5f0de
EA		 3543}
3544
3545static int
76446cac 3546i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
40a5f0de
EA		 3547 uint32_t buffer_count,
3548 struct drm_i915_gem_relocation_entry *relocs)
3549{
3550 uint32_t reloc_count = 0, i;
2bc43b5c 3551 int ret = 0;
40a5f0de 3552
93533c29
CW		 3553 if (relocs == NULL)
3554 return 0;
3555
40a5f0de
EA		 3556 for (i = 0; i < buffer_count; i++) {
3557 struct drm_i915_gem_relocation_entry __user *user_relocs;
2bc43b5c 3558 int unwritten;
40a5f0de
EA		 3559
3560 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3561
2bc43b5c
FM		 3562 unwritten = copy_to_user(user_relocs,
3563 &relocs[reloc_count],
3564 exec_list[i].relocation_count *
3565 sizeof(*relocs));
3566
3567 if (unwritten) {
3568 ret = -EFAULT;
3569 goto err;
40a5f0de
EA		 3570 }
3571
3572 reloc_count += exec_list[i].relocation_count;
3573 }
3574
2bc43b5c 3575err:
8e7d2b2c 3576 drm_free_large(relocs);
40a5f0de
EA		 3577
3578 return ret;
3579}
3580
83d60795 3581static int
76446cac 3582i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
83d60795
CW		 3583 uint64_t exec_offset)
3584{
3585 uint32_t exec_start, exec_len;
3586
3587 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3588 exec_len = (uint32_t) exec->batch_len;
3589
3590 if ((exec_start | exec_len) & 0x7)
3591 return -EINVAL;
3592
3593 if (!exec_start)
3594 return -EINVAL;
3595
3596 return 0;
3597}
3598
e6c3a2a6 3599static int
6b95a207
KH		 3600i915_gem_wait_for_pending_flip(struct drm_device *dev,
3601 struct drm_gem_object **object_list,
3602 int count)
3603{
3604 drm_i915_private_t *dev_priv = dev->dev_private;
3605 struct drm_i915_gem_object *obj_priv;
3606 DEFINE_WAIT(wait);
3607 int i, ret = 0;
3608
3609 for (;;) {
3610 prepare_to_wait(&dev_priv->pending_flip_queue,
3611 &wait, TASK_INTERRUPTIBLE);
3612 for (i = 0; i < count; i++) {
23010e43 3613 obj_priv = to_intel_bo(object_list[i]);
6b95a207
KH		 3614 if (atomic_read(&obj_priv->pending_flip) > 0)
3615 break;
3616 }
3617 if (i == count)
3618 break;
3619
3620 if (!signal_pending(current)) {
3621 mutex_unlock(&dev->struct_mutex);
3622 schedule();
3623 mutex_lock(&dev->struct_mutex);
3624 continue;
3625 }
3626 ret = -ERESTARTSYS;
3627 break;
3628 }
3629 finish_wait(&dev_priv->pending_flip_queue, &wait);
3630
3631 return ret;
3632}
3633
8dc5d147 3634static int
76446cac
JB		 3635i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3636 struct drm_file *file_priv,
3637 struct drm_i915_gem_execbuffer2 *args,
3638 struct drm_i915_gem_exec_object2 *exec_list)
673a394b
EA		 3639{
3640 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3641 struct drm_gem_object **object_list = NULL;
3642 struct drm_gem_object *batch_obj;
b70d11da 3643 struct drm_i915_gem_object *obj_priv;
201361a5 3644 struct drm_clip_rect *cliprects = NULL;
93533c29 3645 struct drm_i915_gem_relocation_entry *relocs = NULL;
8dc5d147 3646 struct drm_i915_gem_request *request = NULL;
30dbf0c0 3647 int ret, ret2, i, pinned = 0;
673a394b 3648 uint64_t exec_offset;
5c12a07e 3649 uint32_t reloc_index;
6b95a207 3650 int pin_tries, flips;
673a394b 3651
852835f3
ZN		 3652 struct intel_ring_buffer *ring = NULL;
3653
30dbf0c0
CW		 3654 ret = i915_gem_check_is_wedged(dev);
3655 if (ret)
3656 return ret;
3657
673a394b
EA		 3658#if WATCH_EXEC
3659 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3660 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3661#endif
d1b851fc
ZN		 3662 if (args->flags & I915_EXEC_BSD) {
3663 if (!HAS_BSD(dev)) {
3664 DRM_ERROR("execbuf with wrong flag\n");
3665 return -EINVAL;
3666 }
3667 ring = &dev_priv->bsd_ring;
3668 } else {
3669 ring = &dev_priv->render_ring;
3670 }
3671
4f481ed2
EA		 3672 if (args->buffer_count < 1) {
3673 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3674 return -EINVAL;
3675 }
c8e0f93a 3676 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
76446cac
JB		 3677 if (object_list == NULL) {
3678 DRM_ERROR("Failed to allocate object list for %d buffers\n",
673a394b
EA		 3679 args->buffer_count);
3680 ret = -ENOMEM;
3681 goto pre_mutex_err;
3682 }
673a394b 3683
201361a5 3684 if (args->num_cliprects != 0) {
9a298b2a
EA		 3685 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3686 GFP_KERNEL);
a40e8d31
OA		 3687 if (cliprects == NULL) {
3688 ret = -ENOMEM;
201361a5 3689 goto pre_mutex_err;
a40e8d31 3690 }
201361a5
EA		 3691
3692 ret = copy_from_user(cliprects,
3693 (struct drm_clip_rect __user *)
3694 (uintptr_t) args->cliprects_ptr,
3695 sizeof(*cliprects) * args->num_cliprects);
3696 if (ret != 0) {
3697 DRM_ERROR("copy %d cliprects failed: %d\n",
3698 args->num_cliprects, ret);
c877cdce 3699 ret = -EFAULT;
201361a5
EA		 3700 goto pre_mutex_err;
3701 }
3702 }
3703
8dc5d147
CW		 3704 request = kzalloc(sizeof(*request), GFP_KERNEL);
3705 if (request == NULL) {
3706 ret = -ENOMEM;
3707 goto pre_mutex_err;
3708 }
3709
40a5f0de
EA		 3710 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3711 &relocs);
3712 if (ret != 0)
3713 goto pre_mutex_err;
3714
76c1dec1
CW		 3715 ret = i915_mutex_lock_interruptible(dev);
3716 if (ret)
3717 goto pre_mutex_err;
673a394b
EA		 3718
3719 i915_verify_inactive(dev, __FILE__, __LINE__);
3720
673a394b 3721 if (dev_priv->mm.suspended) {
673a394b 3722 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3723 ret = -EBUSY;
3724 goto pre_mutex_err;
673a394b
EA		 3725 }
3726
ac94a962 3727 /* Look up object handles */
6b95a207 3728 flips = 0;
673a394b
EA		 3729 for (i = 0; i < args->buffer_count; i++) {
3730 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3731 exec_list[i].handle);
3732 if (object_list[i] == NULL) {
3733 DRM_ERROR("Invalid object handle %d at index %d\n",
3734 exec_list[i].handle, i);
0ce907f8
CW		 3735 /* prevent error path from reading uninitialized data */
3736 args->buffer_count = i + 1;
bf79cb91 3737 ret = -ENOENT;
673a394b
EA		 3738 goto err;
3739 }
b70d11da 3740
23010e43 3741 obj_priv = to_intel_bo(object_list[i]);
b70d11da
KH		 3742 if (obj_priv->in_execbuffer) {
3743 DRM_ERROR("Object %p appears more than once in object list\n",
3744 object_list[i]);
0ce907f8
CW		 3745 /* prevent error path from reading uninitialized data */
3746 args->buffer_count = i + 1;
bf79cb91 3747 ret = -EINVAL;
b70d11da
KH		 3748 goto err;
3749 }
3750 obj_priv->in_execbuffer = true;
6b95a207
KH		 3751 flips += atomic_read(&obj_priv->pending_flip);
3752 }
3753
3754 if (flips > 0) {
3755 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3756 args->buffer_count);
3757 if (ret)
3758 goto err;
ac94a962 3759 }
673a394b 3760
ac94a962
KP		 3761 /* Pin and relocate */
3762 for (pin_tries = 0; ; pin_tries++) {
3763 ret = 0;
40a5f0de
EA		 3764 reloc_index = 0;
3765
ac94a962
KP		 3766 for (i = 0; i < args->buffer_count; i++) {
3767 object_list[i]->pending_read_domains = 0;
3768 object_list[i]->pending_write_domain = 0;
3769 ret = i915_gem_object_pin_and_relocate(object_list[i],
3770 file_priv,
40a5f0de
EA		 3771 &exec_list[i],
3772 &relocs[reloc_index]);
ac94a962
KP		 3773 if (ret)
3774 break;
3775 pinned = i + 1;
40a5f0de 3776 reloc_index += exec_list[i].relocation_count;
ac94a962
KP		 3777 }
3778 /* success */
3779 if (ret == 0)
3780 break;
3781
3782 /* error other than GTT full, or we've already tried again */
2939e1f5 3783 if (ret != -ENOSPC || pin_tries >= 1) {
07f73f69
CW		 3784 if (ret != -ERESTARTSYS) {
3785 unsigned long long total_size = 0;
3d1cc470
CW		 3786 int num_fences = 0;
3787 for (i = 0; i < args->buffer_count; i++) {
43b27f40 3788 obj_priv = to_intel_bo(object_list[i]);
3d1cc470 3789
07f73f69 3790 total_size += object_list[i]->size;
3d1cc470
CW		 3791 num_fences +=
3792 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3793 obj_priv->tiling_mode != I915_TILING_NONE;
3794 }
3795 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
07f73f69 3796 pinned+1, args->buffer_count,
3d1cc470
CW		 3797 total_size, num_fences,
3798 ret);
07f73f69
CW		 3799 DRM_ERROR("%d objects [%d pinned], "
3800 "%d object bytes [%d pinned], "
3801 "%d/%d gtt bytes\n",
3802 atomic_read(&dev->object_count),
3803 atomic_read(&dev->pin_count),
3804 atomic_read(&dev->object_memory),
3805 atomic_read(&dev->pin_memory),
3806 atomic_read(&dev->gtt_memory),
3807 dev->gtt_total);
3808 }
673a394b
EA		 3809 goto err;
3810 }
ac94a962
KP		 3811
3812 /* unpin all of our buffers */
3813 for (i = 0; i < pinned; i++)
3814 i915_gem_object_unpin(object_list[i]);
b1177636 3815 pinned = 0;
ac94a962
KP		 3816
3817 /* evict everyone we can from the aperture */
3818 ret = i915_gem_evict_everything(dev);
07f73f69 3819 if (ret && ret != -ENOSPC)
ac94a962 3820 goto err;
673a394b
EA		 3821 }
3822
3823 /* Set the pending read domains for the batch buffer to COMMAND */
3824 batch_obj = object_list[args->buffer_count-1];
5f26a2c7
CW		 3825 if (batch_obj->pending_write_domain) {
3826 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3827 ret = -EINVAL;
3828 goto err;
3829 }
3830 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
673a394b 3831
83d60795
CW		 3832 /* Sanity check the batch buffer, prior to moving objects */
3833 exec_offset = exec_list[args->buffer_count - 1].offset;
3834 ret = i915_gem_check_execbuffer (args, exec_offset);
3835 if (ret != 0) {
3836 DRM_ERROR("execbuf with invalid offset/length\n");
3837 goto err;
3838 }
3839
673a394b
EA		 3840 i915_verify_inactive(dev, __FILE__, __LINE__);
3841
646f0f6e
KP		 3842 /* Zero the global flush/invalidate flags. These
3843 * will be modified as new domains are computed
3844 * for each object
3845 */
3846 dev->invalidate_domains = 0;
3847 dev->flush_domains = 0;
9220434a 3848 dev_priv->mm.flush_rings = 0;
646f0f6e 3849
673a394b
EA		 3850 for (i = 0; i < args->buffer_count; i++) {
3851 struct drm_gem_object *obj = object_list[i];
673a394b 3852
646f0f6e 3853 /* Compute new gpu domains and update invalidate/flush */
8b0e378a 3854 i915_gem_object_set_to_gpu_domain(obj);
673a394b
EA		 3855 }
3856
3857 i915_verify_inactive(dev, __FILE__, __LINE__);
3858
646f0f6e
KP		 3859 if (dev->invalidate_domains | dev->flush_domains) {
3860#if WATCH_EXEC
3861 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3862 __func__,
3863 dev->invalidate_domains,
3864 dev->flush_domains);
3865#endif
c78ec30b 3866 i915_gem_flush(dev, file_priv,
646f0f6e 3867 dev->invalidate_domains,
9220434a
CW		 3868 dev->flush_domains,
3869 dev_priv->mm.flush_rings);
a6910434
DV		 3870 }
3871
efbeed96
EA		 3872 for (i = 0; i < args->buffer_count; i++) {
3873 struct drm_gem_object *obj = object_list[i];
23010e43 3874 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 3875 uint32_t old_write_domain = obj->write_domain;
efbeed96
EA		 3876
3877 obj->write_domain = obj->pending_write_domain;
99fcb766
DV		 3878 if (obj->write_domain)
3879 list_move_tail(&obj_priv->gpu_write_list,
3880 &dev_priv->mm.gpu_write_list);
3881 else
3882 list_del_init(&obj_priv->gpu_write_list);
3883
1c5d22f7
CW		 3884 trace_i915_gem_object_change_domain(obj,
3885 obj->read_domains,
3886 old_write_domain);
efbeed96
EA		 3887 }
3888
673a394b
EA		 3889 i915_verify_inactive(dev, __FILE__, __LINE__);
3890
3891#if WATCH_COHERENCY
3892 for (i = 0; i < args->buffer_count; i++) {
3893 i915_gem_object_check_coherency(object_list[i],
3894 exec_list[i].handle);
3895 }
3896#endif
3897
673a394b 3898#if WATCH_EXEC
6911a9b8 3899 i915_gem_dump_object(batch_obj,
673a394b
EA		 3900 args->batch_len,
3901 __func__,
3902 ~0);
3903#endif
3904
673a394b 3905 /* Exec the batchbuffer */
852835f3
ZN		 3906 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3907 cliprects, exec_offset);
673a394b
EA		 3908 if (ret) {
3909 DRM_ERROR("dispatch failed %d\n", ret);
3910 goto err;
3911 }
3912
3913 /*
3914 * Ensure that the commands in the batch buffer are
3915 * finished before the interrupt fires
3916 */
8a1a49f9 3917 i915_retire_commands(dev, ring);
673a394b
EA		 3918
3919 i915_verify_inactive(dev, __FILE__, __LINE__);
3920
617dbe27
DV		 3921 for (i = 0; i < args->buffer_count; i++) {
3922 struct drm_gem_object *obj = object_list[i];
3923 obj_priv = to_intel_bo(obj);
3924
3925 i915_gem_object_move_to_active(obj, ring);
3926#if WATCH_LRU
3927 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3928#endif
3929 }
5c12a07e 3930 i915_add_request(dev, file_priv, request, ring);
8dc5d147 3931 request = NULL;
673a394b 3932
673a394b
EA		 3933#if WATCH_LRU
3934 i915_dump_lru(dev, __func__);
3935#endif
3936
3937 i915_verify_inactive(dev, __FILE__, __LINE__);
3938
673a394b 3939err:
aad87dff
JL		 3940 for (i = 0; i < pinned; i++)
3941 i915_gem_object_unpin(object_list[i]);
3942
b70d11da
KH		 3943 for (i = 0; i < args->buffer_count; i++) {
3944 if (object_list[i]) {
23010e43 3945 obj_priv = to_intel_bo(object_list[i]);
b70d11da
KH		 3946 obj_priv->in_execbuffer = false;
3947 }
aad87dff 3948 drm_gem_object_unreference(object_list[i]);
b70d11da 3949 }
673a394b 3950
673a394b
EA		 3951 mutex_unlock(&dev->struct_mutex);
3952
93533c29 3953pre_mutex_err:
40a5f0de
EA		 3954 /* Copy the updated relocations out regardless of current error
3955 * state. Failure to update the relocs would mean that the next
3956 * time userland calls execbuf, it would do so with presumed offset
3957 * state that didn't match the actual object state.
3958 */
3959 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3960 relocs);
3961 if (ret2 != 0) {
3962 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3963
3964 if (ret == 0)
3965 ret = ret2;
3966 }
3967
8e7d2b2c 3968 drm_free_large(object_list);
9a298b2a 3969 kfree(cliprects);
8dc5d147 3970 kfree(request);
673a394b
EA		 3971
3972 return ret;
3973}
3974
76446cac
JB		 3975/*
3976 * Legacy execbuffer just creates an exec2 list from the original exec object
3977 * list array and passes it to the real function.
3978 */
3979int
3980i915_gem_execbuffer(struct drm_device *dev, void *data,
3981 struct drm_file *file_priv)
3982{
3983 struct drm_i915_gem_execbuffer *args = data;
3984 struct drm_i915_gem_execbuffer2 exec2;
3985 struct drm_i915_gem_exec_object *exec_list = NULL;
3986 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3987 int ret, i;
3988
3989#if WATCH_EXEC
3990 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3991 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3992#endif
3993
3994 if (args->buffer_count < 1) {
3995 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3996 return -EINVAL;
3997 }
3998
3999 /* Copy in the exec list from userland */
4000 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4001 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4002 if (exec_list == NULL || exec2_list == NULL) {
4003 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4004 args->buffer_count);
4005 drm_free_large(exec_list);
4006 drm_free_large(exec2_list);
4007 return -ENOMEM;
4008 }
4009 ret = copy_from_user(exec_list,
4010 (struct drm_i915_relocation_entry __user *)
4011 (uintptr_t) args->buffers_ptr,
4012 sizeof(*exec_list) * args->buffer_count);
4013 if (ret != 0) {
4014 DRM_ERROR("copy %d exec entries failed %d\n",
4015 args->buffer_count, ret);
4016 drm_free_large(exec_list);
4017 drm_free_large(exec2_list);
4018 return -EFAULT;
4019 }
4020
4021 for (i = 0; i < args->buffer_count; i++) {
4022 exec2_list[i].handle = exec_list[i].handle;
4023 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4024 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4025 exec2_list[i].alignment = exec_list[i].alignment;
4026 exec2_list[i].offset = exec_list[i].offset;
a6c45cf0 4027 if (INTEL_INFO(dev)->gen < 4)
76446cac
JB		 4028 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4029 else
4030 exec2_list[i].flags = 0;
4031 }
4032
4033 exec2.buffers_ptr = args->buffers_ptr;
4034 exec2.buffer_count = args->buffer_count;
4035 exec2.batch_start_offset = args->batch_start_offset;
4036 exec2.batch_len = args->batch_len;
4037 exec2.DR1 = args->DR1;
4038 exec2.DR4 = args->DR4;
4039 exec2.num_cliprects = args->num_cliprects;
4040 exec2.cliprects_ptr = args->cliprects_ptr;
852835f3 4041 exec2.flags = I915_EXEC_RENDER;
76446cac
JB		 4042
4043 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4044 if (!ret) {
4045 /* Copy the new buffer offsets back to the user's exec list. */
4046 for (i = 0; i < args->buffer_count; i++)
4047 exec_list[i].offset = exec2_list[i].offset;
4048 /* ... and back out to userspace */
4049 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4050 (uintptr_t) args->buffers_ptr,
4051 exec_list,
4052 sizeof(*exec_list) * args->buffer_count);
4053 if (ret) {
4054 ret = -EFAULT;
4055 DRM_ERROR("failed to copy %d exec entries "
4056 "back to user (%d)\n",
4057 args->buffer_count, ret);
4058 }
76446cac
JB		 4059 }
4060
4061 drm_free_large(exec_list);
4062 drm_free_large(exec2_list);
4063 return ret;
4064}
4065
4066int
4067i915_gem_execbuffer2(struct drm_device *dev, void *data,
4068 struct drm_file *file_priv)
4069{
4070 struct drm_i915_gem_execbuffer2 *args = data;
4071 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4072 int ret;
4073
4074#if WATCH_EXEC
4075 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4076 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4077#endif
4078
4079 if (args->buffer_count < 1) {
4080 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4081 return -EINVAL;
4082 }
4083
4084 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4085 if (exec2_list == NULL) {
4086 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4087 args->buffer_count);
4088 return -ENOMEM;
4089 }
4090 ret = copy_from_user(exec2_list,
4091 (struct drm_i915_relocation_entry __user *)
4092 (uintptr_t) args->buffers_ptr,
4093 sizeof(*exec2_list) * args->buffer_count);
4094 if (ret != 0) {
4095 DRM_ERROR("copy %d exec entries failed %d\n",
4096 args->buffer_count, ret);
4097 drm_free_large(exec2_list);
4098 return -EFAULT;
4099 }
4100
4101 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4102 if (!ret) {
4103 /* Copy the new buffer offsets back to the user's exec list. */
4104 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4105 (uintptr_t) args->buffers_ptr,
4106 exec2_list,
4107 sizeof(*exec2_list) * args->buffer_count);
4108 if (ret) {
4109 ret = -EFAULT;
4110 DRM_ERROR("failed to copy %d exec entries "
4111 "back to user (%d)\n",
4112 args->buffer_count, ret);
4113 }
4114 }
4115
4116 drm_free_large(exec2_list);
4117 return ret;
4118}
4119
673a394b
EA		 4120int
4121i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4122{
4123 struct drm_device *dev = obj->dev;
f13d3f73 4124 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 4125 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 4126 int ret;
4127
778c3544
DV		 4128 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4129
673a394b 4130 i915_verify_inactive(dev, __FILE__, __LINE__);
ac0c6b5a
CW		 4131
4132 if (obj_priv->gtt_space != NULL) {
4133 if (alignment == 0)
4134 alignment = i915_gem_get_gtt_alignment(obj);
4135 if (obj_priv->gtt_offset & (alignment - 1)) {
ae7d49d8
CW		 4136 WARN(obj_priv->pin_count,
4137 "bo is already pinned with incorrect alignment:"
4138 " offset=%x, req.alignment=%x\n",
4139 obj_priv->gtt_offset, alignment);
ac0c6b5a
CW		 4140 ret = i915_gem_object_unbind(obj);
4141 if (ret)
4142 return ret;
4143 }
4144 }
4145
673a394b
EA		 4146 if (obj_priv->gtt_space == NULL) {
4147 ret = i915_gem_object_bind_to_gtt(obj, alignment);
9731129c 4148 if (ret)
673a394b 4149 return ret;
22c344e9 4150 }
76446cac 4151
673a394b
EA		 4152 obj_priv->pin_count++;
4153
4154 /* If the object is not active and not pending a flush,
4155 * remove it from the inactive list
4156 */
4157 if (obj_priv->pin_count == 1) {
4158 atomic_inc(&dev->pin_count);
4159 atomic_add(obj->size, &dev->pin_memory);
f13d3f73
CW		 4160 if (!obj_priv->active)
4161 list_move_tail(&obj_priv->list,
4162 &dev_priv->mm.pinned_list);
673a394b
EA		 4163 }
4164 i915_verify_inactive(dev, __FILE__, __LINE__);
4165
4166 return 0;
4167}
4168
4169void
4170i915_gem_object_unpin(struct drm_gem_object *obj)
4171{
4172 struct drm_device *dev = obj->dev;
4173 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 4174 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 4175
4176 i915_verify_inactive(dev, __FILE__, __LINE__);
4177 obj_priv->pin_count--;
4178 BUG_ON(obj_priv->pin_count < 0);
4179 BUG_ON(obj_priv->gtt_space == NULL);
4180
4181 /* If the object is no longer pinned, and is
4182 * neither active nor being flushed, then stick it on
4183 * the inactive list
4184 */
4185 if (obj_priv->pin_count == 0) {
f13d3f73 4186 if (!obj_priv->active)
673a394b
EA		 4187 list_move_tail(&obj_priv->list,
4188 &dev_priv->mm.inactive_list);
4189 atomic_dec(&dev->pin_count);
4190 atomic_sub(obj->size, &dev->pin_memory);
4191 }
4192 i915_verify_inactive(dev, __FILE__, __LINE__);
4193}
4194
4195int
4196i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4197 struct drm_file *file_priv)
4198{
4199 struct drm_i915_gem_pin *args = data;
4200 struct drm_gem_object *obj;
4201 struct drm_i915_gem_object *obj_priv;
4202 int ret;
4203
673a394b
EA		 4204 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4205 if (obj == NULL) {
4206 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4207 args->handle);
bf79cb91 4208 return -ENOENT;
673a394b 4209 }
23010e43 4210 obj_priv = to_intel_bo(obj);
673a394b 4211
76c1dec1
CW		 4212 ret = i915_mutex_lock_interruptible(dev);
4213 if (ret) {
4214 drm_gem_object_unreference_unlocked(obj);
4215 return ret;
4216 }
4217
bb6baf76
CW		 4218 if (obj_priv->madv != I915_MADV_WILLNEED) {
4219 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3ef94daa
CW		 4220 drm_gem_object_unreference(obj);
4221 mutex_unlock(&dev->struct_mutex);
4222 return -EINVAL;
4223 }
4224
79e53945
JB		 4225 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4226 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4227 args->handle);
96dec61d 4228 drm_gem_object_unreference(obj);
673a394b 4229 mutex_unlock(&dev->struct_mutex);
79e53945
JB		 4230 return -EINVAL;
4231 }
4232
4233 obj_priv->user_pin_count++;
4234 obj_priv->pin_filp = file_priv;
4235 if (obj_priv->user_pin_count == 1) {
4236 ret = i915_gem_object_pin(obj, args->alignment);
4237 if (ret != 0) {
4238 drm_gem_object_unreference(obj);
4239 mutex_unlock(&dev->struct_mutex);
4240 return ret;
4241 }
673a394b
EA		 4242 }
4243
4244 /* XXX - flush the CPU caches for pinned objects
4245 * as the X server doesn't manage domains yet
4246 */
e47c68e9 4247 i915_gem_object_flush_cpu_write_domain(obj);
673a394b
EA		 4248 args->offset = obj_priv->gtt_offset;
4249 drm_gem_object_unreference(obj);
4250 mutex_unlock(&dev->struct_mutex);
4251
4252 return 0;
4253}
4254
4255int
4256i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4257 struct drm_file *file_priv)
4258{
4259 struct drm_i915_gem_pin *args = data;
4260 struct drm_gem_object *obj;
79e53945 4261 struct drm_i915_gem_object *obj_priv;
76c1dec1 4262 int ret;
673a394b
EA		 4263
4264 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4265 if (obj == NULL) {
4266 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4267 args->handle);
bf79cb91 4268 return -ENOENT;
673a394b
EA		 4269 }
4270
23010e43 4271 obj_priv = to_intel_bo(obj);
76c1dec1
CW		 4272
4273 ret = i915_mutex_lock_interruptible(dev);
4274 if (ret) {
4275 drm_gem_object_unreference_unlocked(obj);
4276 return ret;
4277 }
4278
79e53945
JB		 4279 if (obj_priv->pin_filp != file_priv) {
4280 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4281 args->handle);
4282 drm_gem_object_unreference(obj);
4283 mutex_unlock(&dev->struct_mutex);
4284 return -EINVAL;
4285 }
4286 obj_priv->user_pin_count--;
4287 if (obj_priv->user_pin_count == 0) {
4288 obj_priv->pin_filp = NULL;
4289 i915_gem_object_unpin(obj);
4290 }
673a394b
EA		 4291
4292 drm_gem_object_unreference(obj);
4293 mutex_unlock(&dev->struct_mutex);
4294 return 0;
4295}
4296
4297int
4298i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4299 struct drm_file *file_priv)
4300{
4301 struct drm_i915_gem_busy *args = data;
4302 struct drm_gem_object *obj;
4303 struct drm_i915_gem_object *obj_priv;
30dbf0c0
CW		 4304 int ret;
4305
673a394b
EA		 4306 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4307 if (obj == NULL) {
4308 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4309 args->handle);
bf79cb91 4310 return -ENOENT;
673a394b
EA		 4311 }
4312
76c1dec1
CW		 4313 ret = i915_mutex_lock_interruptible(dev);
4314 if (ret) {
4315 drm_gem_object_unreference_unlocked(obj);
4316 return ret;
30dbf0c0
CW		 4317 }
4318
0be555b6
CW		 4319 /* Count all active objects as busy, even if they are currently not used
4320 * by the gpu. Users of this interface expect objects to eventually
4321 * become non-busy without any further actions, therefore emit any
4322 * necessary flushes here.
c4de0a5d 4323 */
0be555b6
CW		 4324 obj_priv = to_intel_bo(obj);
4325 args->busy = obj_priv->active;
4326 if (args->busy) {
4327 /* Unconditionally flush objects, even when the gpu still uses this
4328 * object. Userspace calling this function indicates that it wants to
4329 * use this buffer rather sooner than later, so issuing the required
4330 * flush earlier is beneficial.
4331 */
c78ec30b
CW		 4332 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4333 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 4334 obj_priv->ring,
4335 0, obj->write_domain);
0be555b6
CW		 4336
4337 /* Update the active list for the hardware's current position.
4338 * Otherwise this only updates on a delayed timer or when irqs
4339 * are actually unmasked, and our working set ends up being
4340 * larger than required.
4341 */
4342 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4343
4344 args->busy = obj_priv->active;
4345 }
673a394b
EA		 4346
4347 drm_gem_object_unreference(obj);
4348 mutex_unlock(&dev->struct_mutex);
76c1dec1 4349 return 0;
673a394b
EA		 4350}
4351
4352int
4353i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4354 struct drm_file *file_priv)
4355{
4356 return i915_gem_ring_throttle(dev, file_priv);
4357}
4358
3ef94daa
CW		 4359int
4360i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4361 struct drm_file *file_priv)
4362{
4363 struct drm_i915_gem_madvise *args = data;
4364 struct drm_gem_object *obj;
4365 struct drm_i915_gem_object *obj_priv;
76c1dec1 4366 int ret;
3ef94daa
CW		 4367
4368 switch (args->madv) {
4369 case I915_MADV_DONTNEED:
4370 case I915_MADV_WILLNEED:
4371 break;
4372 default:
4373 return -EINVAL;
4374 }
4375
4376 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4377 if (obj == NULL) {
4378 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4379 args->handle);
bf79cb91 4380 return -ENOENT;
3ef94daa 4381 }
23010e43 4382 obj_priv = to_intel_bo(obj);
3ef94daa 4383
76c1dec1
CW		 4384 ret = i915_mutex_lock_interruptible(dev);
4385 if (ret) {
4386 drm_gem_object_unreference_unlocked(obj);
4387 return ret;
4388 }
4389
3ef94daa
CW		 4390 if (obj_priv->pin_count) {
4391 drm_gem_object_unreference(obj);
4392 mutex_unlock(&dev->struct_mutex);
4393
4394 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4395 return -EINVAL;
4396 }
4397
bb6baf76
CW		 4398 if (obj_priv->madv != __I915_MADV_PURGED)
4399 obj_priv->madv = args->madv;
3ef94daa 4400
2d7ef395
CW		 4401 /* if the object is no longer bound, discard its backing storage */
4402 if (i915_gem_object_is_purgeable(obj_priv) &&
4403 obj_priv->gtt_space == NULL)
4404 i915_gem_object_truncate(obj);
4405
bb6baf76
CW		 4406 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4407
3ef94daa
CW		 4408 drm_gem_object_unreference(obj);
4409 mutex_unlock(&dev->struct_mutex);
4410
4411 return 0;
4412}
4413
ac52bc56
DV		 4414struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4415 size_t size)
4416{
c397b908 4417 struct drm_i915_gem_object *obj;
ac52bc56 4418
c397b908
DV		 4419 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4420 if (obj == NULL)
4421 return NULL;
673a394b 4422
c397b908
DV		 4423 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4424 kfree(obj);
4425 return NULL;
4426 }
673a394b 4427
c397b908
DV		 4428 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4429 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
673a394b 4430
c397b908 4431 obj->agp_type = AGP_USER_MEMORY;
62b8b215 4432 obj->base.driver_private = NULL;
c397b908
DV		 4433 obj->fence_reg = I915_FENCE_REG_NONE;
4434 INIT_LIST_HEAD(&obj->list);
4435 INIT_LIST_HEAD(&obj->gpu_write_list);
c397b908 4436 obj->madv = I915_MADV_WILLNEED;
de151cf6 4437
c397b908
DV		 4438 trace_i915_gem_object_create(&obj->base);
4439
4440 return &obj->base;
4441}
4442
4443int i915_gem_init_object(struct drm_gem_object *obj)
4444{
4445 BUG();
de151cf6 4446
673a394b
EA		 4447 return 0;
4448}
4449
be72615b 4450static void i915_gem_free_object_tail(struct drm_gem_object *obj)
673a394b 4451{
de151cf6 4452 struct drm_device *dev = obj->dev;
be72615b 4453 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 4454 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
be72615b 4455 int ret;
673a394b 4456
be72615b
CW		 4457 ret = i915_gem_object_unbind(obj);
4458 if (ret == -ERESTARTSYS) {
4459 list_move(&obj_priv->list,
4460 &dev_priv->mm.deferred_free_list);
4461 return;
4462 }
673a394b 4463
7e616158
CW		 4464 if (obj_priv->mmap_offset)
4465 i915_gem_free_mmap_offset(obj);
de151cf6 4466
c397b908
DV		 4467 drm_gem_object_release(obj);
4468
9a298b2a 4469 kfree(obj_priv->page_cpu_valid);
280b713b 4470 kfree(obj_priv->bit_17);
c397b908 4471 kfree(obj_priv);
673a394b
EA		 4472}
4473
be72615b
CW		 4474void i915_gem_free_object(struct drm_gem_object *obj)
4475{
4476 struct drm_device *dev = obj->dev;
4477 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4478
4479 trace_i915_gem_object_destroy(obj);
4480
4481 while (obj_priv->pin_count > 0)
4482 i915_gem_object_unpin(obj);
4483
4484 if (obj_priv->phys_obj)
4485 i915_gem_detach_phys_object(dev, obj);
4486
4487 i915_gem_free_object_tail(obj);
4488}
4489
29105ccc
CW		 4490int
4491i915_gem_idle(struct drm_device *dev)
4492{
4493 drm_i915_private_t *dev_priv = dev->dev_private;
4494 int ret;
28dfe52a 4495
29105ccc 4496 mutex_lock(&dev->struct_mutex);
1c5d22f7 4497
8187a2b7 4498 if (dev_priv->mm.suspended ||
d1b851fc
ZN		 4499 (dev_priv->render_ring.gem_object == NULL) ||
4500 (HAS_BSD(dev) &&
4501 dev_priv->bsd_ring.gem_object == NULL)) {
29105ccc
CW		 4502 mutex_unlock(&dev->struct_mutex);
4503 return 0;
28dfe52a
EA		 4504 }
4505
29105ccc 4506 ret = i915_gpu_idle(dev);
6dbe2772
KP		 4507 if (ret) {
4508 mutex_unlock(&dev->struct_mutex);
673a394b 4509 return ret;
6dbe2772 4510 }
673a394b 4511
29105ccc
CW		 4512 /* Under UMS, be paranoid and evict. */
4513 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
b47eb4a2 4514 ret = i915_gem_evict_inactive(dev);
29105ccc
CW		 4515 if (ret) {
4516 mutex_unlock(&dev->struct_mutex);
4517 return ret;
4518 }
4519 }
4520
4521 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4522 * We need to replace this with a semaphore, or something.
4523 * And not confound mm.suspended!
4524 */
4525 dev_priv->mm.suspended = 1;
bc0c7f14 4526 del_timer_sync(&dev_priv->hangcheck_timer);
29105ccc
CW		 4527
4528 i915_kernel_lost_context(dev);
6dbe2772 4529 i915_gem_cleanup_ringbuffer(dev);
29105ccc 4530
6dbe2772
KP		 4531 mutex_unlock(&dev->struct_mutex);
4532
29105ccc
CW		 4533 /* Cancel the retire work handler, which should be idle now. */
4534 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4535
673a394b
EA		 4536 return 0;
4537}
4538
e552eb70
JB		 4539/*
4540 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4541 * over cache flushing.
4542 */
8187a2b7 4543static int
e552eb70
JB		 4544i915_gem_init_pipe_control(struct drm_device *dev)
4545{
4546 drm_i915_private_t *dev_priv = dev->dev_private;
4547 struct drm_gem_object *obj;
4548 struct drm_i915_gem_object *obj_priv;
4549 int ret;
4550
34dc4d44 4551 obj = i915_gem_alloc_object(dev, 4096);
e552eb70
JB		 4552 if (obj == NULL) {
4553 DRM_ERROR("Failed to allocate seqno page\n");
4554 ret = -ENOMEM;
4555 goto err;
4556 }
4557 obj_priv = to_intel_bo(obj);
4558 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4559
4560 ret = i915_gem_object_pin(obj, 4096);
4561 if (ret)
4562 goto err_unref;
4563
4564 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4565 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4566 if (dev_priv->seqno_page == NULL)
4567 goto err_unpin;
4568
4569 dev_priv->seqno_obj = obj;
4570 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4571
4572 return 0;
4573
4574err_unpin:
4575 i915_gem_object_unpin(obj);
4576err_unref:
4577 drm_gem_object_unreference(obj);
4578err:
4579 return ret;
4580}
4581
8187a2b7
ZN		 4582
4583static void
e552eb70
JB		 4584i915_gem_cleanup_pipe_control(struct drm_device *dev)
4585{
4586 drm_i915_private_t *dev_priv = dev->dev_private;
4587 struct drm_gem_object *obj;
4588 struct drm_i915_gem_object *obj_priv;
4589
4590 obj = dev_priv->seqno_obj;
4591 obj_priv = to_intel_bo(obj);
4592 kunmap(obj_priv->pages[0]);
4593 i915_gem_object_unpin(obj);
4594 drm_gem_object_unreference(obj);
4595 dev_priv->seqno_obj = NULL;
4596
4597 dev_priv->seqno_page = NULL;
673a394b
EA		 4598}
4599
8187a2b7
ZN		 4600int
4601i915_gem_init_ringbuffer(struct drm_device *dev)
4602{
4603 drm_i915_private_t *dev_priv = dev->dev_private;
4604 int ret;
68f95ba9 4605
8187a2b7
ZN		 4606 if (HAS_PIPE_CONTROL(dev)) {
4607 ret = i915_gem_init_pipe_control(dev);
4608 if (ret)
4609 return ret;
4610 }
68f95ba9 4611
5c1143bb 4612 ret = intel_init_render_ring_buffer(dev);
68f95ba9
CW		 4613 if (ret)
4614 goto cleanup_pipe_control;
4615
4616 if (HAS_BSD(dev)) {
5c1143bb 4617 ret = intel_init_bsd_ring_buffer(dev);
68f95ba9
CW		 4618 if (ret)
4619 goto cleanup_render_ring;
d1b851fc 4620 }
68f95ba9 4621
6f392d54
CW		 4622 dev_priv->next_seqno = 1;
4623
68f95ba9
CW		 4624 return 0;
4625
4626cleanup_render_ring:
4627 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4628cleanup_pipe_control:
4629 if (HAS_PIPE_CONTROL(dev))
4630 i915_gem_cleanup_pipe_control(dev);
8187a2b7
ZN		 4631 return ret;
4632}
4633
4634void
4635i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4636{
4637 drm_i915_private_t *dev_priv = dev->dev_private;
4638
4639 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
d1b851fc
ZN		 4640 if (HAS_BSD(dev))
4641 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
8187a2b7
ZN		 4642 if (HAS_PIPE_CONTROL(dev))
4643 i915_gem_cleanup_pipe_control(dev);
4644}
4645
673a394b
EA		 4646int
4647i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4648 struct drm_file *file_priv)
4649{
4650 drm_i915_private_t *dev_priv = dev->dev_private;
4651 int ret;
4652
79e53945
JB		 4653 if (drm_core_check_feature(dev, DRIVER_MODESET))
4654 return 0;
4655
ba1234d1 4656 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 4657 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 4658 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 4659 }
4660
673a394b 4661 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 4662 dev_priv->mm.suspended = 0;
4663
4664 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 4665 if (ret != 0) {
4666 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 4667 return ret;
d816f6ac 4668 }
9bb2d6f9 4669
852835f3 4670 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
d1b851fc 4671 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
673a394b
EA		 4672 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4673 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
852835f3 4674 BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
d1b851fc 4675 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
673a394b 4676 mutex_unlock(&dev->struct_mutex);
dbb19d30 4677
5f35308b
CW		 4678 ret = drm_irq_install(dev);
4679 if (ret)
4680 goto cleanup_ringbuffer;
dbb19d30 4681
673a394b 4682 return 0;
5f35308b
CW		 4683
4684cleanup_ringbuffer:
4685 mutex_lock(&dev->struct_mutex);
4686 i915_gem_cleanup_ringbuffer(dev);
4687 dev_priv->mm.suspended = 1;
4688 mutex_unlock(&dev->struct_mutex);
4689
4690 return ret;
673a394b
EA		 4691}
4692
4693int
4694i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4695 struct drm_file *file_priv)
4696{
79e53945
JB		 4697 if (drm_core_check_feature(dev, DRIVER_MODESET))
4698 return 0;
4699
dbb19d30 4700 drm_irq_uninstall(dev);
e6890f6f 4701 return i915_gem_idle(dev);
673a394b
EA		 4702}
4703
4704void
4705i915_gem_lastclose(struct drm_device *dev)
4706{
4707 int ret;
673a394b 4708
e806b495
EA		 4709 if (drm_core_check_feature(dev, DRIVER_MODESET))
4710 return;
4711
6dbe2772
KP		 4712 ret = i915_gem_idle(dev);
4713 if (ret)
4714 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 4715}
4716
4717void
4718i915_gem_load(struct drm_device *dev)
4719{
b5aa8a0f 4720 int i;
673a394b
EA		 4721 drm_i915_private_t *dev_priv = dev->dev_private;
4722
673a394b 4723 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
99fcb766 4724 INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
673a394b 4725 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
f13d3f73 4726 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
a09ba7fa 4727 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
be72615b 4728 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
852835f3
ZN		 4729 INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4730 INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
d1b851fc
ZN		 4731 if (HAS_BSD(dev)) {
4732 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4733 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4734 }
007cc8ac
DV		 4735 for (i = 0; i < 16; i++)
4736 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
673a394b
EA		 4737 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4738 i915_gem_retire_work_handler);
30dbf0c0 4739 init_completion(&dev_priv->error_completion);
31169714
CW		 4740 spin_lock(&shrink_list_lock);
4741 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4742 spin_unlock(&shrink_list_lock);
4743
94400120
DA		 4744 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4745 if (IS_GEN3(dev)) {
4746 u32 tmp = I915_READ(MI_ARB_STATE);
4747 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4748 /* arb state is a masked write, so set bit + bit in mask */
4749 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4750 I915_WRITE(MI_ARB_STATE, tmp);
4751 }
4752 }
4753
de151cf6 4754 /* Old X drivers will take 0-2 for front, back, depth buffers */
b397c836
EA		 4755 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4756 dev_priv->fence_reg_start = 3;
de151cf6 4757
a6c45cf0 4758 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 4759 dev_priv->num_fence_regs = 16;
4760 else
4761 dev_priv->num_fence_regs = 8;
4762
b5aa8a0f 4763 /* Initialize fence registers to zero */
a6c45cf0
CW		 4764 switch (INTEL_INFO(dev)->gen) {
4765 case 6:
4766 for (i = 0; i < 16; i++)
4767 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4768 break;
4769 case 5:
4770 case 4:
b5aa8a0f
GH		 4771 for (i = 0; i < 16; i++)
4772 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
a6c45cf0
CW		 4773 break;
4774 case 3:
b5aa8a0f
GH		 4775 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4776 for (i = 0; i < 8; i++)
4777 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
a6c45cf0
CW		 4778 case 2:
4779 for (i = 0; i < 8; i++)
4780 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4781 break;
b5aa8a0f 4782 }
673a394b 4783 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 4784 init_waitqueue_head(&dev_priv->pending_flip_queue);
673a394b 4785}
71acb5eb
DA		 4786
4787/*
4788 * Create a physically contiguous memory object for this object
4789 * e.g. for cursor + overlay regs
4790 */
995b6762
CW		 4791static int i915_gem_init_phys_object(struct drm_device *dev,
4792 int id, int size, int align)
71acb5eb
DA		 4793{
4794 drm_i915_private_t *dev_priv = dev->dev_private;
4795 struct drm_i915_gem_phys_object *phys_obj;
4796 int ret;
4797
4798 if (dev_priv->mm.phys_objs[id - 1] || !size)
4799 return 0;
4800
9a298b2a 4801 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 4802 if (!phys_obj)
4803 return -ENOMEM;
4804
4805 phys_obj->id = id;
4806
6eeefaf3 4807 phys_obj->handle = drm_pci_alloc(dev, size, align);
71acb5eb
DA		 4808 if (!phys_obj->handle) {
4809 ret = -ENOMEM;
4810 goto kfree_obj;
4811 }
4812#ifdef CONFIG_X86
4813 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4814#endif
4815
4816 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4817
4818 return 0;
4819kfree_obj:
9a298b2a 4820 kfree(phys_obj);
71acb5eb
DA		 4821 return ret;
4822}
4823
995b6762 4824static void i915_gem_free_phys_object(struct drm_device *dev, int id)
71acb5eb
DA		 4825{
4826 drm_i915_private_t *dev_priv = dev->dev_private;
4827 struct drm_i915_gem_phys_object *phys_obj;
4828
4829 if (!dev_priv->mm.phys_objs[id - 1])
4830 return;
4831
4832 phys_obj = dev_priv->mm.phys_objs[id - 1];
4833 if (phys_obj->cur_obj) {
4834 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4835 }
4836
4837#ifdef CONFIG_X86
4838 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4839#endif
4840 drm_pci_free(dev, phys_obj->handle);
4841 kfree(phys_obj);
4842 dev_priv->mm.phys_objs[id - 1] = NULL;
4843}
4844
4845void i915_gem_free_all_phys_object(struct drm_device *dev)
4846{
4847 int i;
4848
260883c8 4849 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 4850 i915_gem_free_phys_object(dev, i);
4851}
4852
4853void i915_gem_detach_phys_object(struct drm_device *dev,
4854 struct drm_gem_object *obj)
4855{
4856 struct drm_i915_gem_object *obj_priv;
4857 int i;
4858 int ret;
4859 int page_count;
4860
23010e43 4861 obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4862 if (!obj_priv->phys_obj)
4863 return;
4864
4bdadb97 4865 ret = i915_gem_object_get_pages(obj, 0);
71acb5eb
DA		 4866 if (ret)
4867 goto out;
4868
4869 page_count = obj->size / PAGE_SIZE;
4870
4871 for (i = 0; i < page_count; i++) {
856fa198 4872 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4873 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4874
4875 memcpy(dst, src, PAGE_SIZE);
4876 kunmap_atomic(dst, KM_USER0);
4877 }
856fa198 4878 drm_clflush_pages(obj_priv->pages, page_count);
71acb5eb 4879 drm_agp_chipset_flush(dev);
d78b47b9
CW		 4880
4881 i915_gem_object_put_pages(obj);
71acb5eb
DA		 4882out:
4883 obj_priv->phys_obj->cur_obj = NULL;
4884 obj_priv->phys_obj = NULL;
4885}
4886
4887int
4888i915_gem_attach_phys_object(struct drm_device *dev,
6eeefaf3
CW		 4889 struct drm_gem_object *obj,
4890 int id,
4891 int align)
71acb5eb
DA		 4892{
4893 drm_i915_private_t *dev_priv = dev->dev_private;
4894 struct drm_i915_gem_object *obj_priv;
4895 int ret = 0;
4896 int page_count;
4897 int i;
4898
4899 if (id > I915_MAX_PHYS_OBJECT)
4900 return -EINVAL;
4901
23010e43 4902 obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4903
4904 if (obj_priv->phys_obj) {
4905 if (obj_priv->phys_obj->id == id)
4906 return 0;
4907 i915_gem_detach_phys_object(dev, obj);
4908 }
4909
71acb5eb
DA		 4910 /* create a new object */
4911 if (!dev_priv->mm.phys_objs[id - 1]) {
4912 ret = i915_gem_init_phys_object(dev, id,
6eeefaf3 4913 obj->size, align);
71acb5eb 4914 if (ret) {
aeb565df 4915 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
71acb5eb
DA		 4916 goto out;
4917 }
4918 }
4919
4920 /* bind to the object */
4921 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4922 obj_priv->phys_obj->cur_obj = obj;
4923
4bdadb97 4924 ret = i915_gem_object_get_pages(obj, 0);
71acb5eb
DA		 4925 if (ret) {
4926 DRM_ERROR("failed to get page list\n");
4927 goto out;
4928 }
4929
4930 page_count = obj->size / PAGE_SIZE;
4931
4932 for (i = 0; i < page_count; i++) {
856fa198 4933 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4934 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4935
4936 memcpy(dst, src, PAGE_SIZE);
4937 kunmap_atomic(src, KM_USER0);
4938 }
4939
d78b47b9
CW		 4940 i915_gem_object_put_pages(obj);
4941
71acb5eb
DA		 4942 return 0;
4943out:
4944 return ret;
4945}
4946
4947static int
4948i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4949 struct drm_i915_gem_pwrite *args,
4950 struct drm_file *file_priv)
4951{
23010e43 4952 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4953 void *obj_addr;
4954 int ret;
4955 char __user *user_data;
4956
4957 user_data = (char __user *) (uintptr_t) args->data_ptr;
4958 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4959
44d98a61 4960 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
71acb5eb
DA		 4961 ret = copy_from_user(obj_addr, user_data, args->size);
4962 if (ret)
4963 return -EFAULT;
4964
4965 drm_agp_chipset_flush(dev);
4966 return 0;
4967}
b962442e 4968
f787a5f5 4969void i915_gem_release(struct drm_device *dev, struct drm_file *file)
b962442e 4970{
f787a5f5 4971 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e
EA		 4972
4973 /* Clean up our request list when the client is going away, so that
4974 * later retire_requests won't dereference our soon-to-be-gone
4975 * file_priv.
4976 */
4977 mutex_lock(&dev->struct_mutex);
f787a5f5
CW		 4978 mutex_lock(&file_priv->mutex);
4979 while (!list_empty(&file_priv->mm.request_list)) {
4980 struct drm_i915_gem_request *request;
4981
4982 request = list_first_entry(&file_priv->mm.request_list,
4983 struct drm_i915_gem_request,
4984 client_list);
4985 list_del(&request->client_list);
4986 request->file_priv = NULL;
4987 }
4988 mutex_unlock(&file_priv->mutex);
b962442e
EA		 4989 mutex_unlock(&dev->struct_mutex);
4990}
31169714 4991
1637ef41
CW		 4992static int
4993i915_gpu_is_active(struct drm_device *dev)
4994{
4995 drm_i915_private_t *dev_priv = dev->dev_private;
4996 int lists_empty;
4997
1637ef41 4998 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
852835f3 4999 list_empty(&dev_priv->render_ring.active_list);
d1b851fc
ZN		 5000 if (HAS_BSD(dev))
5001 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
1637ef41
CW		 5002
5003 return !lists_empty;
5004}
5005
31169714 5006static int
7f8275d0 5007i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
31169714
CW		 5008{
5009 drm_i915_private_t *dev_priv, *next_dev;
5010 struct drm_i915_gem_object *obj_priv, *next_obj;
5011 int cnt = 0;
5012 int would_deadlock = 1;
5013
5014 /* "fast-path" to count number of available objects */
5015 if (nr_to_scan == 0) {
5016 spin_lock(&shrink_list_lock);
5017 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5018 struct drm_device *dev = dev_priv->dev;
5019
5020 if (mutex_trylock(&dev->struct_mutex)) {
5021 list_for_each_entry(obj_priv,
5022 &dev_priv->mm.inactive_list,
5023 list)
5024 cnt++;
5025 mutex_unlock(&dev->struct_mutex);
5026 }
5027 }
5028 spin_unlock(&shrink_list_lock);
5029
5030 return (cnt / 100) * sysctl_vfs_cache_pressure;
5031 }
5032
5033 spin_lock(&shrink_list_lock);
5034
1637ef41 5035rescan:
31169714
CW		 5036 /* first scan for clean buffers */
5037 list_for_each_entry_safe(dev_priv, next_dev,
5038 &shrink_list, mm.shrink_list) {
5039 struct drm_device *dev = dev_priv->dev;
5040
5041 if (! mutex_trylock(&dev->struct_mutex))
5042 continue;
5043
5044 spin_unlock(&shrink_list_lock);
b09a1fec 5045 i915_gem_retire_requests(dev);
31169714
CW		 5046
5047 list_for_each_entry_safe(obj_priv, next_obj,
5048 &dev_priv->mm.inactive_list,
5049 list) {
5050 if (i915_gem_object_is_purgeable(obj_priv)) {
a8089e84 5051 i915_gem_object_unbind(&obj_priv->base);
31169714
CW		 5052 if (--nr_to_scan <= 0)
5053 break;
5054 }
5055 }
5056
5057 spin_lock(&shrink_list_lock);
5058 mutex_unlock(&dev->struct_mutex);
5059
963b4836
CW		 5060 would_deadlock = 0;
5061
31169714
CW		 5062 if (nr_to_scan <= 0)
5063 break;
5064 }
5065
5066 /* second pass, evict/count anything still on the inactive list */
5067 list_for_each_entry_safe(dev_priv, next_dev,
5068 &shrink_list, mm.shrink_list) {
5069 struct drm_device *dev = dev_priv->dev;
5070
5071 if (! mutex_trylock(&dev->struct_mutex))
5072 continue;
5073
5074 spin_unlock(&shrink_list_lock);
5075
5076 list_for_each_entry_safe(obj_priv, next_obj,
5077 &dev_priv->mm.inactive_list,
5078 list) {
5079 if (nr_to_scan > 0) {
a8089e84 5080 i915_gem_object_unbind(&obj_priv->base);
31169714
CW		 5081 nr_to_scan--;
5082 } else
5083 cnt++;
5084 }
5085
5086 spin_lock(&shrink_list_lock);
5087 mutex_unlock(&dev->struct_mutex);
5088
5089 would_deadlock = 0;
5090 }
5091
1637ef41
CW		 5092 if (nr_to_scan) {
5093 int active = 0;
5094
5095 /*
5096 * We are desperate for pages, so as a last resort, wait
5097 * for the GPU to finish and discard whatever we can.
5098 * This has a dramatic impact to reduce the number of
5099 * OOM-killer events whilst running the GPU aggressively.
5100 */
5101 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5102 struct drm_device *dev = dev_priv->dev;
5103
5104 if (!mutex_trylock(&dev->struct_mutex))
5105 continue;
5106
5107 spin_unlock(&shrink_list_lock);
5108
5109 if (i915_gpu_is_active(dev)) {
5110 i915_gpu_idle(dev);
5111 active++;
5112 }
5113
5114 spin_lock(&shrink_list_lock);
5115 mutex_unlock(&dev->struct_mutex);
5116 }
5117
5118 if (active)
5119 goto rescan;
5120 }
5121
31169714
CW		 5122 spin_unlock(&shrink_list_lock);
5123
5124 if (would_deadlock)
5125 return -1;
5126 else if (cnt > 0)
5127 return (cnt / 100) * sysctl_vfs_cache_pressure;
5128 else
5129 return 0;
5130}
5131
5132static struct shrinker shrinker = {
5133 .shrink = i915_gem_shrink,
5134 .seeks = DEFAULT_SEEKS,
5135};
5136
5137__init void
5138i915_gem_shrinker_init(void)
5139{
5140 register_shrinker(&shrinker);
5141}
5142
5143__exit void
5144i915_gem_shrinker_exit(void)
5145{
5146 unregister_shrinker(&shrinker);
5147}
Linux kernel - Deliverables
This page took 0.507076 seconds and 5 git commands to generate.