2 * Copyright © 2008-2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include <drm/i915_drm.h>
29 * DOC: fence register handling
31 * Important to avoid confusions: "fences" in the i915 driver are not execution
32 * fences used to track command completion but hardware detiler objects which
33 * wrap a given range of the global GTT. Each platform has only a fairly limited
34 * set of these objects.
36 * Fences are used to detile GTT memory mappings. They're also connected to the
37 * hardware frontbuffer render tracking and hence interract with frontbuffer
38 * conmpression. Furthermore on older platforms fences are required for tiled
39 * objects used by the display engine. They can also be used by the render
40 * engine - they're required for blitter commands and are optional for render
41 * commands. But on gen4+ both display (with the exception of fbc) and rendering
42 * have their own tiling state bits and don't need fences.
44 * Also note that fences only support X and Y tiling and hence can't be used for
45 * the fancier new tiling formats like W, Ys and Yf.
47 * Finally note that because fences are such a restricted resource they're
48 * dynamically associated with objects. Furthermore fence state is committed to
49 * the hardware lazily to avoid unecessary stalls on gen2/3. Therefore code must
50 * explictly call i915_gem_object_get_fence() to synchronize fencing status
51 * for cpu access. Also note that some code wants an unfenced view, for those
52 * cases the fence can be removed forcefully with i915_gem_object_put_fence().
54 * Internally these functions will synchronize with userspace access by removing
55 * CPU ptes into GTT mmaps (not the GTT ptes themselves) as needed.
58 static void i965_write_fence_reg(struct drm_device
*dev
, int reg
,
59 struct drm_i915_gem_object
*obj
)
61 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
63 int fence_pitch_shift
;
65 if (INTEL_INFO(dev
)->gen
>= 6) {
66 fence_reg
= FENCE_REG_SANDYBRIDGE_0
;
67 fence_pitch_shift
= SANDYBRIDGE_FENCE_PITCH_SHIFT
;
69 fence_reg
= FENCE_REG_965_0
;
70 fence_pitch_shift
= I965_FENCE_PITCH_SHIFT
;
75 /* To w/a incoherency with non-atomic 64-bit register updates,
76 * we split the 64-bit update into two 32-bit writes. In order
77 * for a partial fence not to be evaluated between writes, we
78 * precede the update with write to turn off the fence register,
79 * and only enable the fence as the last step.
81 * For extra levels of paranoia, we make sure each step lands
82 * before applying the next step.
84 I915_WRITE(fence_reg
, 0);
85 POSTING_READ(fence_reg
);
88 u32 size
= i915_gem_obj_ggtt_size(obj
);
91 /* Adjust fence size to match tiled area */
92 if (obj
->tiling_mode
!= I915_TILING_NONE
) {
93 uint32_t row_size
= obj
->stride
*
94 (obj
->tiling_mode
== I915_TILING_Y
? 32 : 8);
95 size
= (size
/ row_size
) * row_size
;
98 val
= (uint64_t)((i915_gem_obj_ggtt_offset(obj
) + size
- 4096) &
100 val
|= i915_gem_obj_ggtt_offset(obj
) & 0xfffff000;
101 val
|= (uint64_t)((obj
->stride
/ 128) - 1) << fence_pitch_shift
;
102 if (obj
->tiling_mode
== I915_TILING_Y
)
103 val
|= 1 << I965_FENCE_TILING_Y_SHIFT
;
104 val
|= I965_FENCE_REG_VALID
;
106 I915_WRITE(fence_reg
+ 4, val
>> 32);
107 POSTING_READ(fence_reg
+ 4);
109 I915_WRITE(fence_reg
+ 0, val
);
110 POSTING_READ(fence_reg
);
112 I915_WRITE(fence_reg
+ 4, 0);
113 POSTING_READ(fence_reg
+ 4);
117 static void i915_write_fence_reg(struct drm_device
*dev
, int reg
,
118 struct drm_i915_gem_object
*obj
)
120 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
124 u32 size
= i915_gem_obj_ggtt_size(obj
);
128 WARN((i915_gem_obj_ggtt_offset(obj
) & ~I915_FENCE_START_MASK
) ||
129 (size
& -size
) != size
||
130 (i915_gem_obj_ggtt_offset(obj
) & (size
- 1)),
131 "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
132 i915_gem_obj_ggtt_offset(obj
), obj
->map_and_fenceable
, size
);
134 if (obj
->tiling_mode
== I915_TILING_Y
&& HAS_128_BYTE_Y_TILING(dev
))
139 /* Note: pitch better be a power of two tile widths */
140 pitch_val
= obj
->stride
/ tile_width
;
141 pitch_val
= ffs(pitch_val
) - 1;
143 val
= i915_gem_obj_ggtt_offset(obj
);
144 if (obj
->tiling_mode
== I915_TILING_Y
)
145 val
|= 1 << I830_FENCE_TILING_Y_SHIFT
;
146 val
|= I915_FENCE_SIZE_BITS(size
);
147 val
|= pitch_val
<< I830_FENCE_PITCH_SHIFT
;
148 val
|= I830_FENCE_REG_VALID
;
153 reg
= FENCE_REG_830_0
+ reg
* 4;
155 reg
= FENCE_REG_945_8
+ (reg
- 8) * 4;
157 I915_WRITE(reg
, val
);
161 static void i830_write_fence_reg(struct drm_device
*dev
, int reg
,
162 struct drm_i915_gem_object
*obj
)
164 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
168 u32 size
= i915_gem_obj_ggtt_size(obj
);
171 WARN((i915_gem_obj_ggtt_offset(obj
) & ~I830_FENCE_START_MASK
) ||
172 (size
& -size
) != size
||
173 (i915_gem_obj_ggtt_offset(obj
) & (size
- 1)),
174 "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
175 i915_gem_obj_ggtt_offset(obj
), size
);
177 pitch_val
= obj
->stride
/ 128;
178 pitch_val
= ffs(pitch_val
) - 1;
180 val
= i915_gem_obj_ggtt_offset(obj
);
181 if (obj
->tiling_mode
== I915_TILING_Y
)
182 val
|= 1 << I830_FENCE_TILING_Y_SHIFT
;
183 val
|= I830_FENCE_SIZE_BITS(size
);
184 val
|= pitch_val
<< I830_FENCE_PITCH_SHIFT
;
185 val
|= I830_FENCE_REG_VALID
;
189 I915_WRITE(FENCE_REG_830_0
+ reg
* 4, val
);
190 POSTING_READ(FENCE_REG_830_0
+ reg
* 4);
193 inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object
*obj
)
195 return obj
&& obj
->base
.read_domains
& I915_GEM_DOMAIN_GTT
;
198 static void i915_gem_write_fence(struct drm_device
*dev
, int reg
,
199 struct drm_i915_gem_object
*obj
)
201 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
203 /* Ensure that all CPU reads are completed before installing a fence
204 * and all writes before removing the fence.
206 if (i915_gem_object_needs_mb(dev_priv
->fence_regs
[reg
].obj
))
209 WARN(obj
&& (!obj
->stride
|| !obj
->tiling_mode
),
210 "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
211 obj
->stride
, obj
->tiling_mode
);
214 i830_write_fence_reg(dev
, reg
, obj
);
215 else if (IS_GEN3(dev
))
216 i915_write_fence_reg(dev
, reg
, obj
);
217 else if (INTEL_INFO(dev
)->gen
>= 4)
218 i965_write_fence_reg(dev
, reg
, obj
);
220 /* And similarly be paranoid that no direct access to this region
221 * is reordered to before the fence is installed.
223 if (i915_gem_object_needs_mb(obj
))
227 static inline int fence_number(struct drm_i915_private
*dev_priv
,
228 struct drm_i915_fence_reg
*fence
)
230 return fence
- dev_priv
->fence_regs
;
233 static void i915_gem_object_update_fence(struct drm_i915_gem_object
*obj
,
234 struct drm_i915_fence_reg
*fence
,
237 struct drm_i915_private
*dev_priv
= obj
->base
.dev
->dev_private
;
238 int reg
= fence_number(dev_priv
, fence
);
240 i915_gem_write_fence(obj
->base
.dev
, reg
, enable
? obj
: NULL
);
243 obj
->fence_reg
= reg
;
245 list_move_tail(&fence
->lru_list
, &dev_priv
->mm
.fence_list
);
247 obj
->fence_reg
= I915_FENCE_REG_NONE
;
249 list_del_init(&fence
->lru_list
);
251 obj
->fence_dirty
= false;
254 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object
*obj
)
256 if (obj
->tiling_mode
)
257 i915_gem_release_mmap(obj
);
259 /* As we do not have an associated fence register, we will force
260 * a tiling change if we ever need to acquire one.
262 obj
->fence_dirty
= false;
263 obj
->fence_reg
= I915_FENCE_REG_NONE
;
267 i915_gem_object_wait_fence(struct drm_i915_gem_object
*obj
)
269 if (obj
->last_fenced_req
) {
270 int ret
= i915_wait_request(obj
->last_fenced_req
);
274 i915_gem_request_assign(&obj
->last_fenced_req
, NULL
);
281 * i915_gem_object_put_fence - force-remove fence for an object
282 * @obj: object to map through a fence reg
284 * This function force-removes any fence from the given object, which is useful
285 * if the kernel wants to do untiled GTT access.
289 * 0 on success, negative error code on failure.
292 i915_gem_object_put_fence(struct drm_i915_gem_object
*obj
)
294 struct drm_i915_private
*dev_priv
= obj
->base
.dev
->dev_private
;
295 struct drm_i915_fence_reg
*fence
;
298 ret
= i915_gem_object_wait_fence(obj
);
302 if (obj
->fence_reg
== I915_FENCE_REG_NONE
)
305 fence
= &dev_priv
->fence_regs
[obj
->fence_reg
];
307 if (WARN_ON(fence
->pin_count
))
310 i915_gem_object_fence_lost(obj
);
311 i915_gem_object_update_fence(obj
, fence
, false);
316 static struct drm_i915_fence_reg
*
317 i915_find_fence_reg(struct drm_device
*dev
)
319 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
320 struct drm_i915_fence_reg
*reg
, *avail
;
323 /* First try to find a free reg */
325 for (i
= dev_priv
->fence_reg_start
; i
< dev_priv
->num_fence_regs
; i
++) {
326 reg
= &dev_priv
->fence_regs
[i
];
337 /* None available, try to steal one or wait for a user to finish */
338 list_for_each_entry(reg
, &dev_priv
->mm
.fence_list
, lru_list
) {
346 /* Wait for completion of pending flips which consume fences */
347 if (intel_has_pending_fb_unpin(dev
))
348 return ERR_PTR(-EAGAIN
);
350 return ERR_PTR(-EDEADLK
);
354 * i915_gem_object_get_fence - set up fencing for an object
355 * @obj: object to map through a fence reg
357 * When mapping objects through the GTT, userspace wants to be able to write
358 * to them without having to worry about swizzling if the object is tiled.
359 * This function walks the fence regs looking for a free one for @obj,
360 * stealing one if it can't find any.
362 * It then sets up the reg based on the object's properties: address, pitch
365 * For an untiled surface, this removes any existing fence.
369 * 0 on success, negative error code on failure.
372 i915_gem_object_get_fence(struct drm_i915_gem_object
*obj
)
374 struct drm_device
*dev
= obj
->base
.dev
;
375 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
376 bool enable
= obj
->tiling_mode
!= I915_TILING_NONE
;
377 struct drm_i915_fence_reg
*reg
;
380 /* Have we updated the tiling parameters upon the object and so
381 * will need to serialise the write to the associated fence register?
383 if (obj
->fence_dirty
) {
384 ret
= i915_gem_object_wait_fence(obj
);
389 /* Just update our place in the LRU if our fence is getting reused. */
390 if (obj
->fence_reg
!= I915_FENCE_REG_NONE
) {
391 reg
= &dev_priv
->fence_regs
[obj
->fence_reg
];
392 if (!obj
->fence_dirty
) {
393 list_move_tail(®
->lru_list
,
394 &dev_priv
->mm
.fence_list
);
398 if (WARN_ON(!obj
->map_and_fenceable
))
401 reg
= i915_find_fence_reg(dev
);
406 struct drm_i915_gem_object
*old
= reg
->obj
;
408 ret
= i915_gem_object_wait_fence(old
);
412 i915_gem_object_fence_lost(old
);
417 i915_gem_object_update_fence(obj
, reg
, enable
);
423 * i915_gem_object_pin_fence - pin fencing state
424 * @obj: object to pin fencing for
426 * This pins the fencing state (whether tiled or untiled) to make sure the
427 * object is ready to be used as a scanout target. Fencing status must be
428 * synchronize first by calling i915_gem_object_get_fence():
430 * The resulting fence pin reference must be released again with
431 * i915_gem_object_unpin_fence().
435 * True if the object has a fence, false otherwise.
438 i915_gem_object_pin_fence(struct drm_i915_gem_object
*obj
)
440 if (obj
->fence_reg
!= I915_FENCE_REG_NONE
) {
441 struct drm_i915_private
*dev_priv
= obj
->base
.dev
->dev_private
;
442 struct i915_vma
*ggtt_vma
= i915_gem_obj_to_ggtt(obj
);
445 dev_priv
->fence_regs
[obj
->fence_reg
].pin_count
>
446 ggtt_vma
->pin_count
);
447 dev_priv
->fence_regs
[obj
->fence_reg
].pin_count
++;
454 * i915_gem_object_unpin_fence - unpin fencing state
455 * @obj: object to unpin fencing for
457 * This releases the fence pin reference acquired through
458 * i915_gem_object_pin_fence. It will handle both objects with and without an
459 * attached fence correctly, callers do not need to distinguish this.
462 i915_gem_object_unpin_fence(struct drm_i915_gem_object
*obj
)
464 if (obj
->fence_reg
!= I915_FENCE_REG_NONE
) {
465 struct drm_i915_private
*dev_priv
= obj
->base
.dev
->dev_private
;
466 WARN_ON(dev_priv
->fence_regs
[obj
->fence_reg
].pin_count
<= 0);
467 dev_priv
->fence_regs
[obj
->fence_reg
].pin_count
--;
472 * i915_gem_restore_fences - restore fence state
475 * Restore the hw fence state to match the software tracking again, to be called
476 * after a gpu reset and on resume.
478 void i915_gem_restore_fences(struct drm_device
*dev
)
480 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
483 for (i
= 0; i
< dev_priv
->num_fence_regs
; i
++) {
484 struct drm_i915_fence_reg
*reg
= &dev_priv
->fence_regs
[i
];
487 * Commit delayed tiling changes if we have an object still
488 * attached to the fence, otherwise just clear the fence.
491 i915_gem_object_update_fence(reg
->obj
, reg
,
492 reg
->obj
->tiling_mode
);
494 i915_gem_write_fence(dev
, i
, NULL
);
500 * DOC: tiling swizzling details
502 * The idea behind tiling is to increase cache hit rates by rearranging
503 * pixel data so that a group of pixel accesses are in the same cacheline.
504 * Performance improvement from doing this on the back/depth buffer are on
507 * Intel architectures make this somewhat more complicated, though, by
508 * adjustments made to addressing of data when the memory is in interleaved
509 * mode (matched pairs of DIMMS) to improve memory bandwidth.
510 * For interleaved memory, the CPU sends every sequential 64 bytes
511 * to an alternate memory channel so it can get the bandwidth from both.
513 * The GPU also rearranges its accesses for increased bandwidth to interleaved
514 * memory, and it matches what the CPU does for non-tiled. However, when tiled
515 * it does it a little differently, since one walks addresses not just in the
516 * X direction but also Y. So, along with alternating channels when bit
517 * 6 of the address flips, it also alternates when other bits flip -- Bits 9
518 * (every 512 bytes, an X tile scanline) and 10 (every two X tile scanlines)
519 * are common to both the 915 and 965-class hardware.
521 * The CPU also sometimes XORs in higher bits as well, to improve
522 * bandwidth doing strided access like we do so frequently in graphics. This
523 * is called "Channel XOR Randomization" in the MCH documentation. The result
524 * is that the CPU is XORing in either bit 11 or bit 17 to bit 6 of its address
527 * All of this bit 6 XORing has an effect on our memory management,
528 * as we need to make sure that the 3d driver can correctly address object
531 * If we don't have interleaved memory, all tiling is safe and no swizzling is
534 * When bit 17 is XORed in, we simply refuse to tile at all. Bit
535 * 17 is not just a page offset, so as we page an objet out and back in,
536 * individual pages in it will have different bit 17 addresses, resulting in
537 * each 64 bytes being swapped with its neighbor!
539 * Otherwise, if interleaved, we have to tell the 3d driver what the address
540 * swizzling it needs to do is, since it's writing with the CPU to the pages
541 * (bit 6 and potentially bit 11 XORed in), and the GPU is reading from the
542 * pages (bit 6, 9, and 10 XORed in), resulting in a cumulative bit swizzling
543 * required by the CPU of XORing in bit 6, 9, 10, and potentially 11, in order
544 * to match what the GPU expects.
548 * i915_gem_detect_bit_6_swizzle - detect bit 6 swizzling pattern
551 * Detects bit 6 swizzling of address lookup between IGD access and CPU
552 * access through main memory.
555 i915_gem_detect_bit_6_swizzle(struct drm_device
*dev
)
557 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
558 uint32_t swizzle_x
= I915_BIT_6_SWIZZLE_UNKNOWN
;
559 uint32_t swizzle_y
= I915_BIT_6_SWIZZLE_UNKNOWN
;
561 if (INTEL_INFO(dev
)->gen
>= 8 || IS_VALLEYVIEW(dev
)) {
563 * On BDW+, swizzling is not used. We leave the CPU memory
564 * controller in charge of optimizing memory accesses without
565 * the extra address manipulation GPU side.
567 * VLV and CHV don't have GPU swizzling.
569 swizzle_x
= I915_BIT_6_SWIZZLE_NONE
;
570 swizzle_y
= I915_BIT_6_SWIZZLE_NONE
;
571 } else if (INTEL_INFO(dev
)->gen
>= 6) {
572 if (dev_priv
->preserve_bios_swizzle
) {
573 if (I915_READ(DISP_ARB_CTL
) &
574 DISP_TILE_SURFACE_SWIZZLING
) {
575 swizzle_x
= I915_BIT_6_SWIZZLE_9_10
;
576 swizzle_y
= I915_BIT_6_SWIZZLE_9
;
578 swizzle_x
= I915_BIT_6_SWIZZLE_NONE
;
579 swizzle_y
= I915_BIT_6_SWIZZLE_NONE
;
582 uint32_t dimm_c0
, dimm_c1
;
583 dimm_c0
= I915_READ(MAD_DIMM_C0
);
584 dimm_c1
= I915_READ(MAD_DIMM_C1
);
585 dimm_c0
&= MAD_DIMM_A_SIZE_MASK
| MAD_DIMM_B_SIZE_MASK
;
586 dimm_c1
&= MAD_DIMM_A_SIZE_MASK
| MAD_DIMM_B_SIZE_MASK
;
587 /* Enable swizzling when the channels are populated
588 * with identically sized dimms. We don't need to check
589 * the 3rd channel because no cpu with gpu attached
590 * ships in that configuration. Also, swizzling only
591 * makes sense for 2 channels anyway. */
592 if (dimm_c0
== dimm_c1
) {
593 swizzle_x
= I915_BIT_6_SWIZZLE_9_10
;
594 swizzle_y
= I915_BIT_6_SWIZZLE_9
;
596 swizzle_x
= I915_BIT_6_SWIZZLE_NONE
;
597 swizzle_y
= I915_BIT_6_SWIZZLE_NONE
;
600 } else if (IS_GEN5(dev
)) {
601 /* On Ironlake whatever DRAM config, GPU always do
602 * same swizzling setup.
604 swizzle_x
= I915_BIT_6_SWIZZLE_9_10
;
605 swizzle_y
= I915_BIT_6_SWIZZLE_9
;
606 } else if (IS_GEN2(dev
)) {
607 /* As far as we know, the 865 doesn't have these bit 6
610 swizzle_x
= I915_BIT_6_SWIZZLE_NONE
;
611 swizzle_y
= I915_BIT_6_SWIZZLE_NONE
;
612 } else if (IS_MOBILE(dev
) || (IS_GEN3(dev
) && !IS_G33(dev
))) {
615 /* On 9xx chipsets, channel interleave by the CPU is
616 * determined by DCC. For single-channel, neither the CPU
617 * nor the GPU do swizzling. For dual channel interleaved,
618 * the GPU's interleave is bit 9 and 10 for X tiled, and bit
619 * 9 for Y tiled. The CPU's interleave is independent, and
620 * can be based on either bit 11 (haven't seen this yet) or
623 dcc
= I915_READ(DCC
);
624 switch (dcc
& DCC_ADDRESSING_MODE_MASK
) {
625 case DCC_ADDRESSING_MODE_SINGLE_CHANNEL
:
626 case DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC
:
627 swizzle_x
= I915_BIT_6_SWIZZLE_NONE
;
628 swizzle_y
= I915_BIT_6_SWIZZLE_NONE
;
630 case DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED
:
631 if (dcc
& DCC_CHANNEL_XOR_DISABLE
) {
632 /* This is the base swizzling by the GPU for
635 swizzle_x
= I915_BIT_6_SWIZZLE_9_10
;
636 swizzle_y
= I915_BIT_6_SWIZZLE_9
;
637 } else if ((dcc
& DCC_CHANNEL_XOR_BIT_17
) == 0) {
638 /* Bit 11 swizzling by the CPU in addition. */
639 swizzle_x
= I915_BIT_6_SWIZZLE_9_10_11
;
640 swizzle_y
= I915_BIT_6_SWIZZLE_9_11
;
642 /* Bit 17 swizzling by the CPU in addition. */
643 swizzle_x
= I915_BIT_6_SWIZZLE_9_10_17
;
644 swizzle_y
= I915_BIT_6_SWIZZLE_9_17
;
649 /* check for L-shaped memory aka modified enhanced addressing */
651 uint32_t ddc2
= I915_READ(DCC2
);
653 if (!(ddc2
& DCC2_MODIFIED_ENHANCED_DISABLE
))
654 dev_priv
->quirks
|= QUIRK_PIN_SWIZZLED_PAGES
;
657 if (dcc
== 0xffffffff) {
658 DRM_ERROR("Couldn't read from MCHBAR. "
659 "Disabling tiling.\n");
660 swizzle_x
= I915_BIT_6_SWIZZLE_UNKNOWN
;
661 swizzle_y
= I915_BIT_6_SWIZZLE_UNKNOWN
;
664 /* The 965, G33, and newer, have a very flexible memory
665 * configuration. It will enable dual-channel mode
666 * (interleaving) on as much memory as it can, and the GPU
667 * will additionally sometimes enable different bit 6
668 * swizzling for tiled objects from the CPU.
670 * Here's what I found on the G965:
671 * slot fill memory size swizzling
672 * 0A 0B 1A 1B 1-ch 2-ch
674 * 512 0 512 0 16 1008 X
675 * 512 0 0 512 16 1008 X
676 * 0 512 0 512 16 1008 X
677 * 1024 1024 1024 0 2048 1024 O
679 * We could probably detect this based on either the DRB
680 * matching, which was the case for the swizzling required in
681 * the table above, or from the 1-ch value being less than
682 * the minimum size of a rank.
684 if (I915_READ16(C0DRB3
) != I915_READ16(C1DRB3
)) {
685 swizzle_x
= I915_BIT_6_SWIZZLE_NONE
;
686 swizzle_y
= I915_BIT_6_SWIZZLE_NONE
;
688 swizzle_x
= I915_BIT_6_SWIZZLE_9_10
;
689 swizzle_y
= I915_BIT_6_SWIZZLE_9
;
693 dev_priv
->mm
.bit_6_swizzle_x
= swizzle_x
;
694 dev_priv
->mm
.bit_6_swizzle_y
= swizzle_y
;
698 * Swap every 64 bytes of this page around, to account for it having a new
699 * bit 17 of its physical address and therefore being interpreted differently
703 i915_gem_swizzle_page(struct page
*page
)
711 for (i
= 0; i
< PAGE_SIZE
; i
+= 128) {
712 memcpy(temp
, &vaddr
[i
], 64);
713 memcpy(&vaddr
[i
], &vaddr
[i
+ 64], 64);
714 memcpy(&vaddr
[i
+ 64], temp
, 64);
721 * i915_gem_object_do_bit_17_swizzle - fixup bit 17 swizzling
722 * @obj: i915 GEM buffer object
724 * This function fixes up the swizzling in case any page frame number for this
725 * object has changed in bit 17 since that state has been saved with
726 * i915_gem_object_save_bit_17_swizzle().
728 * This is called when pinning backing storage again, since the kernel is free
729 * to move unpinned backing storage around (either by directly moving pages or
730 * by swapping them out and back in again).
733 i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object
*obj
)
735 struct sg_page_iter sg_iter
;
738 if (obj
->bit_17
== NULL
)
742 for_each_sg_page(obj
->pages
->sgl
, &sg_iter
, obj
->pages
->nents
, 0) {
743 struct page
*page
= sg_page_iter_page(&sg_iter
);
744 char new_bit_17
= page_to_phys(page
) >> 17;
745 if ((new_bit_17
& 0x1) !=
746 (test_bit(i
, obj
->bit_17
) != 0)) {
747 i915_gem_swizzle_page(page
);
748 set_page_dirty(page
);
755 * i915_gem_object_save_bit_17_swizzle - save bit 17 swizzling
756 * @obj: i915 GEM buffer object
758 * This function saves the bit 17 of each page frame number so that swizzling
759 * can be fixed up later on with i915_gem_object_do_bit_17_swizzle(). This must
760 * be called before the backing storage can be unpinned.
763 i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object
*obj
)
765 struct sg_page_iter sg_iter
;
766 int page_count
= obj
->base
.size
>> PAGE_SHIFT
;
769 if (obj
->bit_17
== NULL
) {
770 obj
->bit_17
= kcalloc(BITS_TO_LONGS(page_count
),
771 sizeof(long), GFP_KERNEL
);
772 if (obj
->bit_17
== NULL
) {
773 DRM_ERROR("Failed to allocate memory for bit 17 "
780 for_each_sg_page(obj
->pages
->sgl
, &sg_iter
, obj
->pages
->nents
, 0) {
781 if (page_to_phys(sg_page_iter_page(&sg_iter
)) & (1 << 17))
782 __set_bit(i
, obj
->bit_17
);
784 __clear_bit(i
, obj
->bit_17
);