X-Git-Url: http://git.efficios.com/?a=blobdiff_plain;f=drivers%2Fgpu%2Fdrm%2Fi915%2Fintel_pm.c;h=87ab6544dc48f46fa9fb8e5cee142256b6375cda;hb=da7e29bd5b6dfab9b64eeffa7816fdcf00048d14;hp=3657ab43c8fd1e20fc9deb10856166b4c35ec0d4;hpb=7c06abcb16e1a03270f70725d0c62e8cf818cf48;p=deliverable%2Flinux.git diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 3657ab43c8fd..87ab6544dc48 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -30,7 +30,9 @@ #include "intel_drv.h" #include "../../../platform/x86/intel_ips.h" #include +#include #include +#include /** * RC6 is a special power stage which allows the GPU to enter an very @@ -86,7 +88,7 @@ static void i8xx_disable_fbc(struct drm_device *dev) DRM_DEBUG_KMS("disabled FBC\n"); } -static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval) +static void i8xx_enable_fbc(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -95,37 +97,44 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval) struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int cfb_pitch; - int plane, i; - u32 fbc_ctl, fbc_ctl2; + int i; + u32 fbc_ctl; cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE; if (fb->pitches[0] < cfb_pitch) cfb_pitch = fb->pitches[0]; - /* FBC_CTL wants 64B units */ - cfb_pitch = (cfb_pitch / 64) - 1; - plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB; + /* FBC_CTL wants 32B or 64B units */ + if (IS_GEN2(dev)) + cfb_pitch = (cfb_pitch / 32) - 1; + else + cfb_pitch = (cfb_pitch / 64) - 1; /* Clear old tags */ for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++) I915_WRITE(FBC_TAG + (i * 4), 0); - /* Set it up... */ - fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE; - fbc_ctl2 |= plane; - I915_WRITE(FBC_CONTROL2, fbc_ctl2); - I915_WRITE(FBC_FENCE_OFF, crtc->y); + if (IS_GEN4(dev)) { + u32 fbc_ctl2; + + /* Set it up... */ + fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE; + fbc_ctl2 |= FBC_CTL_PLANE(intel_crtc->plane); + I915_WRITE(FBC_CONTROL2, fbc_ctl2); + I915_WRITE(FBC_FENCE_OFF, crtc->y); + } /* enable it... */ - fbc_ctl = FBC_CTL_EN | FBC_CTL_PERIODIC; + fbc_ctl = I915_READ(FBC_CONTROL); + fbc_ctl &= 0x3fff << FBC_CTL_INTERVAL_SHIFT; + fbc_ctl |= FBC_CTL_EN | FBC_CTL_PERIODIC; if (IS_I945GM(dev)) fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */ fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT; - fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT; fbc_ctl |= obj->fence_reg; I915_WRITE(FBC_CONTROL, fbc_ctl); - DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c, ", + DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c\n", cfb_pitch, crtc->y, plane_name(intel_crtc->plane)); } @@ -136,7 +145,7 @@ static bool i8xx_fbc_enabled(struct drm_device *dev) return I915_READ(FBC_CONTROL) & FBC_CTL_EN; } -static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval) +static void g4x_enable_fbc(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -144,21 +153,19 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval) struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB; - unsigned long stall_watermark = 200; u32 dpfc_ctl; - dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X; + dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane) | DPFC_SR_EN; + if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) + dpfc_ctl |= DPFC_CTL_LIMIT_2X; + else + dpfc_ctl |= DPFC_CTL_LIMIT_1X; dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg; - I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY); - I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN | - (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) | - (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT)); I915_WRITE(DPFC_FENCE_YOFF, crtc->y); /* enable it... */ - I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN); + I915_WRITE(DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane)); } @@ -191,7 +198,11 @@ static void sandybridge_blit_fbc_update(struct drm_device *dev) u32 blt_ecoskpd; /* Make sure blitter notifies FBC of writes */ - gen6_gt_force_wake_get(dev_priv); + + /* Blitter is part of Media powerwell on VLV. No impact of + * his param in other platforms for now */ + gen6_gt_force_wake_get(dev_priv, FORCEWAKE_MEDIA); + blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD); blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY << GEN6_BLITTER_LOCK_SHIFT; @@ -202,10 +213,11 @@ static void sandybridge_blit_fbc_update(struct drm_device *dev) GEN6_BLITTER_LOCK_SHIFT); I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd); POSTING_READ(GEN6_BLITTER_ECOSKPD); - gen6_gt_force_wake_put(dev_priv); + + gen6_gt_force_wake_put(dev_priv, FORCEWAKE_MEDIA); } -static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval) +static void ironlake_enable_fbc(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -213,21 +225,17 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval) struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB; - unsigned long stall_watermark = 200; u32 dpfc_ctl; - dpfc_ctl = I915_READ(ILK_DPFC_CONTROL); - dpfc_ctl &= DPFC_RESERVED; - dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X); - /* Set persistent mode for front-buffer rendering, ala X. */ - dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE; - dpfc_ctl |= (DPFC_CTL_FENCE_EN | obj->fence_reg); - I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY); - - I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN | - (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) | - (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT)); + dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane); + if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) + dpfc_ctl |= DPFC_CTL_LIMIT_2X; + else + dpfc_ctl |= DPFC_CTL_LIMIT_1X; + dpfc_ctl |= DPFC_CTL_FENCE_EN; + if (IS_GEN5(dev)) + dpfc_ctl |= obj->fence_reg; + I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y); I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) | ILK_FBC_RT_VALID); /* enable it... */ @@ -265,7 +273,7 @@ static bool ironlake_fbc_enabled(struct drm_device *dev) return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN; } -static void gen7_enable_fbc(struct drm_crtc *crtc, unsigned long interval) +static void gen7_enable_fbc(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -273,12 +281,16 @@ static void gen7_enable_fbc(struct drm_crtc *crtc, unsigned long interval) struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + u32 dpfc_ctl; - I915_WRITE(IVB_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj)); + dpfc_ctl = IVB_DPFC_CTL_PLANE(intel_crtc->plane); + if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) + dpfc_ctl |= DPFC_CTL_LIMIT_2X; + else + dpfc_ctl |= DPFC_CTL_LIMIT_1X; + dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN; - I915_WRITE(ILK_DPFC_CONTROL, DPFC_CTL_EN | DPFC_CTL_LIMIT_1X | - IVB_DPFC_CTL_FENCE_EN | - intel_crtc->plane << IVB_DPFC_CTL_PLANE_SHIFT); + I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); if (IS_IVYBRIDGE(dev)) { /* WaFbcAsynchFlipDisableFbcQueue:ivb */ @@ -295,7 +307,7 @@ static void gen7_enable_fbc(struct drm_crtc *crtc, unsigned long interval) sandybridge_blit_fbc_update(dev); - DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane); + DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane)); } bool intel_fbc_enabled(struct drm_device *dev) @@ -322,8 +334,7 @@ static void intel_fbc_work_fn(struct work_struct *__work) * the prior work. */ if (work->crtc->fb == work->fb) { - dev_priv->display.enable_fbc(work->crtc, - work->interval); + dev_priv->display.enable_fbc(work->crtc); dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane; dev_priv->fbc.fb_id = work->crtc->fb->base.id; @@ -360,7 +371,7 @@ static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv) dev_priv->fbc.fbc_work = NULL; } -static void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval) +static void intel_enable_fbc(struct drm_crtc *crtc) { struct intel_fbc_work *work; struct drm_device *dev = crtc->dev; @@ -374,13 +385,12 @@ static void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval) work = kzalloc(sizeof(*work), GFP_KERNEL); if (work == NULL) { DRM_ERROR("Failed to allocate FBC work structure\n"); - dev_priv->display.enable_fbc(crtc, interval); + dev_priv->display.enable_fbc(crtc); return; } work->crtc = crtc; work->fb = crtc->fb; - work->interval = interval; INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn); dev_priv->fbc.fbc_work = work; @@ -454,12 +464,12 @@ void intel_update_fbc(struct drm_device *dev) const struct drm_display_mode *adjusted_mode; unsigned int max_width, max_height; - if (!I915_HAS_FBC(dev)) { + if (!HAS_FBC(dev)) { set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED); return; } - if (!i915_powersave) { + if (!i915.powersave) { if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM)) DRM_DEBUG_KMS("fbc disabled per module param\n"); return; @@ -498,13 +508,13 @@ void intel_update_fbc(struct drm_device *dev) obj = intel_fb->obj; adjusted_mode = &intel_crtc->config.adjusted_mode; - if (i915_enable_fbc < 0 && + if (i915.enable_fbc < 0 && INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) { if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT)) DRM_DEBUG_KMS("disabled per chip default\n"); goto out_disable; } - if (!i915_enable_fbc) { + if (!i915.enable_fbc) { if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM)) DRM_DEBUG_KMS("fbc disabled per module param\n"); goto out_disable; @@ -530,10 +540,10 @@ void intel_update_fbc(struct drm_device *dev) DRM_DEBUG_KMS("mode too large for compression, disabling\n"); goto out_disable; } - if ((IS_I915GM(dev) || IS_I945GM(dev) || IS_HASWELL(dev)) && - intel_crtc->plane != 0) { + if ((INTEL_INFO(dev)->gen < 4 || IS_HASWELL(dev)) && + intel_crtc->plane != PLANE_A) { if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE)) - DRM_DEBUG_KMS("plane not 0, disabling compression\n"); + DRM_DEBUG_KMS("plane not A, disabling compression\n"); goto out_disable; } @@ -595,7 +605,7 @@ void intel_update_fbc(struct drm_device *dev) intel_disable_fbc(dev); } - intel_enable_fbc(crtc, 500); + intel_enable_fbc(crtc); dev_priv->fbc.no_fbc_reason = FBC_OK; return; @@ -817,7 +827,7 @@ static int i9xx_get_fifo_size(struct drm_device *dev, int plane) return size; } -static int i85x_get_fifo_size(struct drm_device *dev, int plane) +static int i830_get_fifo_size(struct drm_device *dev, int plane) { struct drm_i915_private *dev_priv = dev->dev_private; uint32_t dsparb = I915_READ(DSPARB); @@ -850,21 +860,6 @@ static int i845_get_fifo_size(struct drm_device *dev, int plane) return size; } -static int i830_get_fifo_size(struct drm_device *dev, int plane) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t dsparb = I915_READ(DSPARB); - int size; - - size = dsparb & 0x7f; - size >>= 1; /* Convert to cachelines */ - - DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb, - plane ? "B" : "A", size); - - return size; -} - /* Pineview has different values for various configs */ static const struct intel_watermark_params pineview_display_wm = { PINEVIEW_DISPLAY_FIFO, @@ -943,14 +938,14 @@ static const struct intel_watermark_params i915_wm_info = { 2, I915_FIFO_LINE_SIZE }; -static const struct intel_watermark_params i855_wm_info = { +static const struct intel_watermark_params i830_wm_info = { I855GM_FIFO_SIZE, I915_MAX_WM, 1, 2, I830_FIFO_LINE_SIZE }; -static const struct intel_watermark_params i830_wm_info = { +static const struct intel_watermark_params i845_wm_info = { I830_FIFO_SIZE, I915_MAX_WM, 1, @@ -958,65 +953,6 @@ static const struct intel_watermark_params i830_wm_info = { I830_FIFO_LINE_SIZE }; -static const struct intel_watermark_params ironlake_display_wm_info = { - ILK_DISPLAY_FIFO, - ILK_DISPLAY_MAXWM, - ILK_DISPLAY_DFTWM, - 2, - ILK_FIFO_LINE_SIZE -}; -static const struct intel_watermark_params ironlake_cursor_wm_info = { - ILK_CURSOR_FIFO, - ILK_CURSOR_MAXWM, - ILK_CURSOR_DFTWM, - 2, - ILK_FIFO_LINE_SIZE -}; -static const struct intel_watermark_params ironlake_display_srwm_info = { - ILK_DISPLAY_SR_FIFO, - ILK_DISPLAY_MAX_SRWM, - ILK_DISPLAY_DFT_SRWM, - 2, - ILK_FIFO_LINE_SIZE -}; -static const struct intel_watermark_params ironlake_cursor_srwm_info = { - ILK_CURSOR_SR_FIFO, - ILK_CURSOR_MAX_SRWM, - ILK_CURSOR_DFT_SRWM, - 2, - ILK_FIFO_LINE_SIZE -}; - -static const struct intel_watermark_params sandybridge_display_wm_info = { - SNB_DISPLAY_FIFO, - SNB_DISPLAY_MAXWM, - SNB_DISPLAY_DFTWM, - 2, - SNB_FIFO_LINE_SIZE -}; -static const struct intel_watermark_params sandybridge_cursor_wm_info = { - SNB_CURSOR_FIFO, - SNB_CURSOR_MAXWM, - SNB_CURSOR_DFTWM, - 2, - SNB_FIFO_LINE_SIZE -}; -static const struct intel_watermark_params sandybridge_display_srwm_info = { - SNB_DISPLAY_SR_FIFO, - SNB_DISPLAY_MAX_SRWM, - SNB_DISPLAY_DFT_SRWM, - 2, - SNB_FIFO_LINE_SIZE -}; -static const struct intel_watermark_params sandybridge_cursor_srwm_info = { - SNB_CURSOR_SR_FIFO, - SNB_CURSOR_MAX_SRWM, - SNB_CURSOR_DFT_SRWM, - 2, - SNB_FIFO_LINE_SIZE -}; - - /** * intel_calculate_wm - calculate watermark level * @clock_in_khz: pixel clock @@ -1567,7 +1503,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) else if (!IS_GEN2(dev)) wm_info = &i915_wm_info; else - wm_info = &i855_wm_info; + wm_info = &i830_wm_info; fifo_size = dev_priv->display.get_fifo_size(dev, 0); crtc = intel_get_crtc_for_plane(dev, 0); @@ -1615,7 +1551,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) if (IS_I945G(dev) || IS_I945GM(dev)) I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK | 0); else if (IS_I915GM(dev)) - I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN); + I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_SELF_EN)); /* Calc sr entries for one plane configs */ if (HAS_FW_BLC(dev) && enabled) { @@ -1667,14 +1603,14 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK | FW_BLC_SELF_EN); else if (IS_I915GM(dev)) - I915_WRITE(INSTPM, I915_READ(INSTPM) | INSTPM_SELF_EN); + I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_SELF_EN)); DRM_DEBUG_KMS("memory self refresh enabled\n"); } else DRM_DEBUG_KMS("memory self refresh disabled\n"); } } -static void i830_update_wm(struct drm_crtc *unused_crtc) +static void i845_update_wm(struct drm_crtc *unused_crtc) { struct drm_device *dev = unused_crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1689,7 +1625,7 @@ static void i830_update_wm(struct drm_crtc *unused_crtc) adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, - &i830_wm_info, + &i845_wm_info, dev_priv->display.get_fifo_size(dev, 0), 4, latency_ns); fwater_lo = I915_READ(FW_BLC) & ~0xfff; @@ -1700,423 +1636,6 @@ static void i830_update_wm(struct drm_crtc *unused_crtc) I915_WRITE(FW_BLC, fwater_lo); } -/* - * Check the wm result. - * - * If any calculated watermark values is larger than the maximum value that - * can be programmed into the associated watermark register, that watermark - * must be disabled. - */ -static bool ironlake_check_srwm(struct drm_device *dev, int level, - int fbc_wm, int display_wm, int cursor_wm, - const struct intel_watermark_params *display, - const struct intel_watermark_params *cursor) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - - DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d," - " cursor %d\n", level, display_wm, fbc_wm, cursor_wm); - - if (fbc_wm > SNB_FBC_MAX_SRWM) { - DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n", - fbc_wm, SNB_FBC_MAX_SRWM, level); - - /* fbc has it's own way to disable FBC WM */ - I915_WRITE(DISP_ARB_CTL, - I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS); - return false; - } else if (INTEL_INFO(dev)->gen >= 6) { - /* enable FBC WM (except on ILK, where it must remain off) */ - I915_WRITE(DISP_ARB_CTL, - I915_READ(DISP_ARB_CTL) & ~DISP_FBC_WM_DIS); - } - - if (display_wm > display->max_wm) { - DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n", - display_wm, SNB_DISPLAY_MAX_SRWM, level); - return false; - } - - if (cursor_wm > cursor->max_wm) { - DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n", - cursor_wm, SNB_CURSOR_MAX_SRWM, level); - return false; - } - - if (!(fbc_wm || display_wm || cursor_wm)) { - DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level); - return false; - } - - return true; -} - -/* - * Compute watermark values of WM[1-3], - */ -static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane, - int latency_ns, - const struct intel_watermark_params *display, - const struct intel_watermark_params *cursor, - int *fbc_wm, int *display_wm, int *cursor_wm) -{ - struct drm_crtc *crtc; - const struct drm_display_mode *adjusted_mode; - unsigned long line_time_us; - int hdisplay, htotal, pixel_size, clock; - int line_count, line_size; - int small, large; - int entries; - - if (!latency_ns) { - *fbc_wm = *display_wm = *cursor_wm = 0; - return false; - } - - crtc = intel_get_crtc_for_plane(dev, plane); - adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; - clock = adjusted_mode->crtc_clock; - htotal = adjusted_mode->crtc_htotal; - hdisplay = to_intel_crtc(crtc)->config.pipe_src_w; - pixel_size = crtc->fb->bits_per_pixel / 8; - - line_time_us = (htotal * 1000) / clock; - line_count = (latency_ns / line_time_us + 1000) / 1000; - line_size = hdisplay * pixel_size; - - /* Use the minimum of the small and large buffer method for primary */ - small = ((clock * pixel_size / 1000) * latency_ns) / 1000; - large = line_count * line_size; - - entries = DIV_ROUND_UP(min(small, large), display->cacheline_size); - *display_wm = entries + display->guard_size; - - /* - * Spec says: - * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2 - */ - *fbc_wm = DIV_ROUND_UP(*display_wm * 64, line_size) + 2; - - /* calculate the self-refresh watermark for display cursor */ - entries = line_count * pixel_size * 64; - entries = DIV_ROUND_UP(entries, cursor->cacheline_size); - *cursor_wm = entries + cursor->guard_size; - - return ironlake_check_srwm(dev, level, - *fbc_wm, *display_wm, *cursor_wm, - display, cursor); -} - -static void ironlake_update_wm(struct drm_crtc *crtc) -{ - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - int fbc_wm, plane_wm, cursor_wm; - unsigned int enabled; - - enabled = 0; - if (g4x_compute_wm0(dev, PIPE_A, - &ironlake_display_wm_info, - dev_priv->wm.pri_latency[0] * 100, - &ironlake_cursor_wm_info, - dev_priv->wm.cur_latency[0] * 100, - &plane_wm, &cursor_wm)) { - I915_WRITE(WM0_PIPEA_ILK, - (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm); - DRM_DEBUG_KMS("FIFO watermarks For pipe A -" - " plane %d, " "cursor: %d\n", - plane_wm, cursor_wm); - enabled |= 1 << PIPE_A; - } - - if (g4x_compute_wm0(dev, PIPE_B, - &ironlake_display_wm_info, - dev_priv->wm.pri_latency[0] * 100, - &ironlake_cursor_wm_info, - dev_priv->wm.cur_latency[0] * 100, - &plane_wm, &cursor_wm)) { - I915_WRITE(WM0_PIPEB_ILK, - (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm); - DRM_DEBUG_KMS("FIFO watermarks For pipe B -" - " plane %d, cursor: %d\n", - plane_wm, cursor_wm); - enabled |= 1 << PIPE_B; - } - - /* - * Calculate and update the self-refresh watermark only when one - * display plane is used. - */ - I915_WRITE(WM3_LP_ILK, 0); - I915_WRITE(WM2_LP_ILK, 0); - I915_WRITE(WM1_LP_ILK, 0); - - if (!single_plane_enabled(enabled)) - return; - enabled = ffs(enabled) - 1; - - /* WM1 */ - if (!ironlake_compute_srwm(dev, 1, enabled, - dev_priv->wm.pri_latency[1] * 500, - &ironlake_display_srwm_info, - &ironlake_cursor_srwm_info, - &fbc_wm, &plane_wm, &cursor_wm)) - return; - - I915_WRITE(WM1_LP_ILK, - WM1_LP_SR_EN | - (dev_priv->wm.pri_latency[1] << WM1_LP_LATENCY_SHIFT) | - (fbc_wm << WM1_LP_FBC_SHIFT) | - (plane_wm << WM1_LP_SR_SHIFT) | - cursor_wm); - - /* WM2 */ - if (!ironlake_compute_srwm(dev, 2, enabled, - dev_priv->wm.pri_latency[2] * 500, - &ironlake_display_srwm_info, - &ironlake_cursor_srwm_info, - &fbc_wm, &plane_wm, &cursor_wm)) - return; - - I915_WRITE(WM2_LP_ILK, - WM2_LP_EN | - (dev_priv->wm.pri_latency[2] << WM1_LP_LATENCY_SHIFT) | - (fbc_wm << WM1_LP_FBC_SHIFT) | - (plane_wm << WM1_LP_SR_SHIFT) | - cursor_wm); - - /* - * WM3 is unsupported on ILK, probably because we don't have latency - * data for that power state - */ -} - -static void sandybridge_update_wm(struct drm_crtc *crtc) -{ - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - int latency = dev_priv->wm.pri_latency[0] * 100; /* In unit 0.1us */ - u32 val; - int fbc_wm, plane_wm, cursor_wm; - unsigned int enabled; - - enabled = 0; - if (g4x_compute_wm0(dev, PIPE_A, - &sandybridge_display_wm_info, latency, - &sandybridge_cursor_wm_info, latency, - &plane_wm, &cursor_wm)) { - val = I915_READ(WM0_PIPEA_ILK); - val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK); - I915_WRITE(WM0_PIPEA_ILK, val | - ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm)); - DRM_DEBUG_KMS("FIFO watermarks For pipe A -" - " plane %d, " "cursor: %d\n", - plane_wm, cursor_wm); - enabled |= 1 << PIPE_A; - } - - if (g4x_compute_wm0(dev, PIPE_B, - &sandybridge_display_wm_info, latency, - &sandybridge_cursor_wm_info, latency, - &plane_wm, &cursor_wm)) { - val = I915_READ(WM0_PIPEB_ILK); - val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK); - I915_WRITE(WM0_PIPEB_ILK, val | - ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm)); - DRM_DEBUG_KMS("FIFO watermarks For pipe B -" - " plane %d, cursor: %d\n", - plane_wm, cursor_wm); - enabled |= 1 << PIPE_B; - } - - /* - * Calculate and update the self-refresh watermark only when one - * display plane is used. - * - * SNB support 3 levels of watermark. - * - * WM1/WM2/WM2 watermarks have to be enabled in the ascending order, - * and disabled in the descending order - * - */ - I915_WRITE(WM3_LP_ILK, 0); - I915_WRITE(WM2_LP_ILK, 0); - I915_WRITE(WM1_LP_ILK, 0); - - if (!single_plane_enabled(enabled) || - dev_priv->sprite_scaling_enabled) - return; - enabled = ffs(enabled) - 1; - - /* WM1 */ - if (!ironlake_compute_srwm(dev, 1, enabled, - dev_priv->wm.pri_latency[1] * 500, - &sandybridge_display_srwm_info, - &sandybridge_cursor_srwm_info, - &fbc_wm, &plane_wm, &cursor_wm)) - return; - - I915_WRITE(WM1_LP_ILK, - WM1_LP_SR_EN | - (dev_priv->wm.pri_latency[1] << WM1_LP_LATENCY_SHIFT) | - (fbc_wm << WM1_LP_FBC_SHIFT) | - (plane_wm << WM1_LP_SR_SHIFT) | - cursor_wm); - - /* WM2 */ - if (!ironlake_compute_srwm(dev, 2, enabled, - dev_priv->wm.pri_latency[2] * 500, - &sandybridge_display_srwm_info, - &sandybridge_cursor_srwm_info, - &fbc_wm, &plane_wm, &cursor_wm)) - return; - - I915_WRITE(WM2_LP_ILK, - WM2_LP_EN | - (dev_priv->wm.pri_latency[2] << WM1_LP_LATENCY_SHIFT) | - (fbc_wm << WM1_LP_FBC_SHIFT) | - (plane_wm << WM1_LP_SR_SHIFT) | - cursor_wm); - - /* WM3 */ - if (!ironlake_compute_srwm(dev, 3, enabled, - dev_priv->wm.pri_latency[3] * 500, - &sandybridge_display_srwm_info, - &sandybridge_cursor_srwm_info, - &fbc_wm, &plane_wm, &cursor_wm)) - return; - - I915_WRITE(WM3_LP_ILK, - WM3_LP_EN | - (dev_priv->wm.pri_latency[3] << WM1_LP_LATENCY_SHIFT) | - (fbc_wm << WM1_LP_FBC_SHIFT) | - (plane_wm << WM1_LP_SR_SHIFT) | - cursor_wm); -} - -static void ivybridge_update_wm(struct drm_crtc *crtc) -{ - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - int latency = dev_priv->wm.pri_latency[0] * 100; /* In unit 0.1us */ - u32 val; - int fbc_wm, plane_wm, cursor_wm; - int ignore_fbc_wm, ignore_plane_wm, ignore_cursor_wm; - unsigned int enabled; - - enabled = 0; - if (g4x_compute_wm0(dev, PIPE_A, - &sandybridge_display_wm_info, latency, - &sandybridge_cursor_wm_info, latency, - &plane_wm, &cursor_wm)) { - val = I915_READ(WM0_PIPEA_ILK); - val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK); - I915_WRITE(WM0_PIPEA_ILK, val | - ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm)); - DRM_DEBUG_KMS("FIFO watermarks For pipe A -" - " plane %d, " "cursor: %d\n", - plane_wm, cursor_wm); - enabled |= 1 << PIPE_A; - } - - if (g4x_compute_wm0(dev, PIPE_B, - &sandybridge_display_wm_info, latency, - &sandybridge_cursor_wm_info, latency, - &plane_wm, &cursor_wm)) { - val = I915_READ(WM0_PIPEB_ILK); - val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK); - I915_WRITE(WM0_PIPEB_ILK, val | - ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm)); - DRM_DEBUG_KMS("FIFO watermarks For pipe B -" - " plane %d, cursor: %d\n", - plane_wm, cursor_wm); - enabled |= 1 << PIPE_B; - } - - if (g4x_compute_wm0(dev, PIPE_C, - &sandybridge_display_wm_info, latency, - &sandybridge_cursor_wm_info, latency, - &plane_wm, &cursor_wm)) { - val = I915_READ(WM0_PIPEC_IVB); - val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK); - I915_WRITE(WM0_PIPEC_IVB, val | - ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm)); - DRM_DEBUG_KMS("FIFO watermarks For pipe C -" - " plane %d, cursor: %d\n", - plane_wm, cursor_wm); - enabled |= 1 << PIPE_C; - } - - /* - * Calculate and update the self-refresh watermark only when one - * display plane is used. - * - * SNB support 3 levels of watermark. - * - * WM1/WM2/WM2 watermarks have to be enabled in the ascending order, - * and disabled in the descending order - * - */ - I915_WRITE(WM3_LP_ILK, 0); - I915_WRITE(WM2_LP_ILK, 0); - I915_WRITE(WM1_LP_ILK, 0); - - if (!single_plane_enabled(enabled) || - dev_priv->sprite_scaling_enabled) - return; - enabled = ffs(enabled) - 1; - - /* WM1 */ - if (!ironlake_compute_srwm(dev, 1, enabled, - dev_priv->wm.pri_latency[1] * 500, - &sandybridge_display_srwm_info, - &sandybridge_cursor_srwm_info, - &fbc_wm, &plane_wm, &cursor_wm)) - return; - - I915_WRITE(WM1_LP_ILK, - WM1_LP_SR_EN | - (dev_priv->wm.pri_latency[1] << WM1_LP_LATENCY_SHIFT) | - (fbc_wm << WM1_LP_FBC_SHIFT) | - (plane_wm << WM1_LP_SR_SHIFT) | - cursor_wm); - - /* WM2 */ - if (!ironlake_compute_srwm(dev, 2, enabled, - dev_priv->wm.pri_latency[2] * 500, - &sandybridge_display_srwm_info, - &sandybridge_cursor_srwm_info, - &fbc_wm, &plane_wm, &cursor_wm)) - return; - - I915_WRITE(WM2_LP_ILK, - WM2_LP_EN | - (dev_priv->wm.pri_latency[2] << WM1_LP_LATENCY_SHIFT) | - (fbc_wm << WM1_LP_FBC_SHIFT) | - (plane_wm << WM1_LP_SR_SHIFT) | - cursor_wm); - - /* WM3, note we have to correct the cursor latency */ - if (!ironlake_compute_srwm(dev, 3, enabled, - dev_priv->wm.pri_latency[3] * 500, - &sandybridge_display_srwm_info, - &sandybridge_cursor_srwm_info, - &fbc_wm, &plane_wm, &ignore_cursor_wm) || - !ironlake_compute_srwm(dev, 3, enabled, - dev_priv->wm.cur_latency[3] * 500, - &sandybridge_display_srwm_info, - &sandybridge_cursor_srwm_info, - &ignore_fbc_wm, &ignore_plane_wm, &cursor_wm)) - return; - - I915_WRITE(WM3_LP_ILK, - WM3_LP_EN | - (dev_priv->wm.pri_latency[3] << WM1_LP_LATENCY_SHIFT) | - (fbc_wm << WM1_LP_FBC_SHIFT) | - (plane_wm << WM1_LP_SR_SHIFT) | - cursor_wm); -} - static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev, struct drm_crtc *crtc) { @@ -2185,7 +1704,7 @@ static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels, return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2; } -struct hsw_pipe_wm_parameters { +struct ilk_pipe_wm_parameters { bool active; uint32_t pipe_htotal; uint32_t pixel_rate; @@ -2194,7 +1713,7 @@ struct hsw_pipe_wm_parameters { struct intel_plane_wm_parameters cur; }; -struct hsw_wm_maximums { +struct ilk_wm_maximums { uint16_t pri; uint16_t spr; uint16_t cur; @@ -2212,7 +1731,7 @@ struct intel_wm_config { * For both WM_PIPE and WM_LP. * mem_value must be in 0.1us units. */ -static uint32_t ilk_compute_pri_wm(const struct hsw_pipe_wm_parameters *params, +static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params, uint32_t mem_value, bool is_lp) { @@ -2241,7 +1760,7 @@ static uint32_t ilk_compute_pri_wm(const struct hsw_pipe_wm_parameters *params, * For both WM_PIPE and WM_LP. * mem_value must be in 0.1us units. */ -static uint32_t ilk_compute_spr_wm(const struct hsw_pipe_wm_parameters *params, +static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params, uint32_t mem_value) { uint32_t method1, method2; @@ -2264,7 +1783,7 @@ static uint32_t ilk_compute_spr_wm(const struct hsw_pipe_wm_parameters *params, * For both WM_PIPE and WM_LP. * mem_value must be in 0.1us units. */ -static uint32_t ilk_compute_cur_wm(const struct hsw_pipe_wm_parameters *params, +static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params, uint32_t mem_value) { if (!params->active || !params->cur.enabled) @@ -2278,7 +1797,7 @@ static uint32_t ilk_compute_cur_wm(const struct hsw_pipe_wm_parameters *params, } /* Only for WM_LP. */ -static uint32_t ilk_compute_fbc_wm(const struct hsw_pipe_wm_parameters *params, +static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params, uint32_t pri_val) { if (!params->active || !params->pri.enabled) @@ -2370,7 +1889,7 @@ static unsigned int ilk_cursor_wm_max(const struct drm_device *dev, } /* Calculate the maximum FBC watermark */ -static unsigned int ilk_fbc_wm_max(struct drm_device *dev) +static unsigned int ilk_fbc_wm_max(const struct drm_device *dev) { /* max that registers can hold */ if (INTEL_INFO(dev)->gen >= 8) @@ -2379,11 +1898,11 @@ static unsigned int ilk_fbc_wm_max(struct drm_device *dev) return 15; } -static void ilk_compute_wm_maximums(struct drm_device *dev, +static void ilk_compute_wm_maximums(const struct drm_device *dev, int level, const struct intel_wm_config *config, enum intel_ddb_partitioning ddb_partitioning, - struct hsw_wm_maximums *max) + struct ilk_wm_maximums *max) { max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false); max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true); @@ -2392,7 +1911,7 @@ static void ilk_compute_wm_maximums(struct drm_device *dev, } static bool ilk_validate_wm_level(int level, - const struct hsw_wm_maximums *max, + const struct ilk_wm_maximums *max, struct intel_wm_level *result) { bool ret; @@ -2432,9 +1951,9 @@ static bool ilk_validate_wm_level(int level, return ret; } -static void ilk_compute_wm_level(struct drm_i915_private *dev_priv, +static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, int level, - const struct hsw_pipe_wm_parameters *p, + const struct ilk_pipe_wm_parameters *p, struct intel_wm_level *result) { uint16_t pri_latency = dev_priv->wm.pri_latency[level]; @@ -2482,7 +2001,7 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[5]) { struct drm_i915_private *dev_priv = dev->dev_private; - if (IS_HASWELL(dev)) { + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { uint64_t sskpd = I915_READ64(MCH_SSKPD); wm[0] = (sskpd >> 56) & 0xFF; @@ -2530,7 +2049,7 @@ static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5]) static int ilk_wm_max_level(const struct drm_device *dev) { /* how many WM levels are we expecting */ - if (IS_HASWELL(dev)) + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) return 4; else if (INTEL_INFO(dev)->gen >= 6) return 3; @@ -2582,8 +2101,8 @@ static void intel_setup_wm_latency(struct drm_device *dev) intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency); } -static void hsw_compute_wm_parameters(struct drm_crtc *crtc, - struct hsw_pipe_wm_parameters *p, +static void ilk_compute_wm_parameters(struct drm_crtc *crtc, + struct ilk_pipe_wm_parameters *p, struct intel_wm_config *config) { struct drm_device *dev = crtc->dev; @@ -2593,7 +2112,7 @@ static void hsw_compute_wm_parameters(struct drm_crtc *crtc, p->active = intel_crtc_active(crtc); if (p->active) { - p->pipe_htotal = intel_crtc->config.adjusted_mode.htotal; + p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal; p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc); p->pri.bytes_per_pixel = crtc->fb->bits_per_pixel / 8; p->cur.bytes_per_pixel = 4; @@ -2620,11 +2139,11 @@ static void hsw_compute_wm_parameters(struct drm_crtc *crtc, /* Compute new watermarks for the pipe */ static bool intel_compute_pipe_wm(struct drm_crtc *crtc, - const struct hsw_pipe_wm_parameters *params, + const struct ilk_pipe_wm_parameters *params, struct intel_pipe_wm *pipe_wm) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + const struct drm_i915_private *dev_priv = dev->dev_private; int level, max_level = ilk_wm_max_level(dev); /* LP0 watermark maximums depend on this pipe alone */ struct intel_wm_config config = { @@ -2632,16 +2151,25 @@ static bool intel_compute_pipe_wm(struct drm_crtc *crtc, .sprites_enabled = params->spr.enabled, .sprites_scaled = params->spr.scaled, }; - struct hsw_wm_maximums max; + struct ilk_wm_maximums max; /* LP0 watermarks always use 1/2 DDB partitioning */ ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max); + /* ILK/SNB: LP2+ watermarks only w/o sprites */ + if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled) + max_level = 1; + + /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */ + if (params->spr.scaled) + max_level = 0; + for (level = 0; level <= max_level; level++) ilk_compute_wm_level(dev_priv, level, params, &pipe_wm->wm[level]); - pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc); + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc); /* At least LP0 must be valid */ return ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]); @@ -2676,12 +2204,19 @@ static void ilk_merge_wm_level(struct drm_device *dev, * Merge all low power watermarks for all active pipes. */ static void ilk_wm_merge(struct drm_device *dev, - const struct hsw_wm_maximums *max, + const struct intel_wm_config *config, + const struct ilk_wm_maximums *max, struct intel_pipe_wm *merged) { int level, max_level = ilk_wm_max_level(dev); - merged->fbc_wm_enabled = true; + /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */ + if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) && + config->num_pipes_active > 1) + return; + + /* ILK: FBC WM must be disabled always */ + merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6; /* merge each WM1+ level */ for (level = 1; level <= max_level; level++) { @@ -2701,6 +2236,20 @@ static void ilk_wm_merge(struct drm_device *dev, wm->fbc_val = 0; } } + + /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */ + /* + * FIXME this is racy. FBC might get enabled later. + * What we should check here is whether FBC can be + * enabled sometime later. + */ + if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) { + for (level = 2; level <= max_level; level++) { + struct intel_wm_level *wm = &merged->wm[level]; + + wm->enable = false; + } + } } static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm) @@ -2709,10 +2258,21 @@ static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm) return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable); } -static void hsw_compute_wm_results(struct drm_device *dev, +/* The value we need to program into the WM_LPx latency field */ +static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + return 2 * level; + else + return dev_priv->wm.pri_latency[level]; +} + +static void ilk_compute_wm_results(struct drm_device *dev, const struct intel_pipe_wm *merged, enum intel_ddb_partitioning partitioning, - struct hsw_wm_values *results) + struct ilk_wm_values *results) { struct intel_crtc *intel_crtc; int level, wm_lp; @@ -2731,7 +2291,7 @@ static void hsw_compute_wm_results(struct drm_device *dev, break; results->wm_lp[wm_lp - 1] = WM3_LP_EN | - ((level * 2) << WM1_LP_LATENCY_SHIFT) | + (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) | (r->pri_val << WM1_LP_SR_SHIFT) | r->cur_val; @@ -2742,7 +2302,11 @@ static void hsw_compute_wm_results(struct drm_device *dev, results->wm_lp[wm_lp - 1] |= r->fbc_val << WM1_LP_FBC_SHIFT; - results->wm_lp_spr[wm_lp - 1] = r->spr_val; + if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) { + WARN_ON(wm_lp != 1); + results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val; + } else + results->wm_lp_spr[wm_lp - 1] = r->spr_val; } /* LP0 register values */ @@ -2765,7 +2329,7 @@ static void hsw_compute_wm_results(struct drm_device *dev, /* Find the result with the highest level enabled. Check for enable_fbc_wm in * case both are at the same level. Prefer r1 in case they're the same. */ -static struct intel_pipe_wm *hsw_find_best_result(struct drm_device *dev, +static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev, struct intel_pipe_wm *r1, struct intel_pipe_wm *r2) { @@ -2800,8 +2364,8 @@ static struct intel_pipe_wm *hsw_find_best_result(struct drm_device *dev, #define WM_DIRTY_DDB (1 << 25) static unsigned int ilk_compute_wm_dirty(struct drm_device *dev, - const struct hsw_wm_values *old, - const struct hsw_wm_values *new) + const struct ilk_wm_values *old, + const struct ilk_wm_values *new) { unsigned int dirty = 0; enum pipe pipe; @@ -2851,27 +2415,53 @@ static unsigned int ilk_compute_wm_dirty(struct drm_device *dev, return dirty; } +static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, + unsigned int dirty) +{ + struct ilk_wm_values *previous = &dev_priv->wm.hw; + bool changed = false; + + if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) { + previous->wm_lp[2] &= ~WM1_LP_SR_EN; + I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]); + changed = true; + } + if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) { + previous->wm_lp[1] &= ~WM1_LP_SR_EN; + I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]); + changed = true; + } + if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) { + previous->wm_lp[0] &= ~WM1_LP_SR_EN; + I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]); + changed = true; + } + + /* + * Don't touch WM1S_LP_EN here. + * Doing so could cause underruns. + */ + + return changed; +} + /* * The spec says we shouldn't write when we don't need, because every write * causes WMs to be re-evaluated, expending some power. */ -static void hsw_write_wm_values(struct drm_i915_private *dev_priv, - struct hsw_wm_values *results) +static void ilk_write_wm_values(struct drm_i915_private *dev_priv, + struct ilk_wm_values *results) { - struct hsw_wm_values *previous = &dev_priv->wm.hw; + struct drm_device *dev = dev_priv->dev; + struct ilk_wm_values *previous = &dev_priv->wm.hw; unsigned int dirty; uint32_t val; - dirty = ilk_compute_wm_dirty(dev_priv->dev, previous, results); + dirty = ilk_compute_wm_dirty(dev, previous, results); if (!dirty) return; - if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != 0) - I915_WRITE(WM3_LP_ILK, 0); - if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != 0) - I915_WRITE(WM2_LP_ILK, 0); - if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != 0) - I915_WRITE(WM1_LP_ILK, 0); + _ilk_disable_lp_wm(dev_priv, dirty); if (dirty & WM_DIRTY_PIPE(PIPE_A)) I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]); @@ -2888,12 +2478,21 @@ static void hsw_write_wm_values(struct drm_i915_private *dev_priv, I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]); if (dirty & WM_DIRTY_DDB) { - val = I915_READ(WM_MISC); - if (results->partitioning == INTEL_DDB_PART_1_2) - val &= ~WM_MISC_DATA_PARTITION_5_6; - else - val |= WM_MISC_DATA_PARTITION_5_6; - I915_WRITE(WM_MISC, val); + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { + val = I915_READ(WM_MISC); + if (results->partitioning == INTEL_DDB_PART_1_2) + val &= ~WM_MISC_DATA_PARTITION_5_6; + else + val |= WM_MISC_DATA_PARTITION_5_6; + I915_WRITE(WM_MISC, val); + } else { + val = I915_READ(DISP_ARB_CTL2); + if (results->partitioning == INTEL_DDB_PART_1_2) + val &= ~DISP_DATA_PARTITION_5_6; + else + val |= DISP_DATA_PARTITION_5_6; + I915_WRITE(DISP_ARB_CTL2, val); + } } if (dirty & WM_DIRTY_FBC) { @@ -2905,37 +2504,48 @@ static void hsw_write_wm_values(struct drm_i915_private *dev_priv, I915_WRITE(DISP_ARB_CTL, val); } - if (dirty & WM_DIRTY_LP(1) && previous->wm_lp_spr[0] != results->wm_lp_spr[0]) + if (dirty & WM_DIRTY_LP(1) && + previous->wm_lp_spr[0] != results->wm_lp_spr[0]) I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]); - if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1]) - I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]); - if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2]) - I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]); - if (dirty & WM_DIRTY_LP(1) && results->wm_lp[0] != 0) - I915_WRITE(WM1_LP_ILK, results->wm_lp[0]); - if (dirty & WM_DIRTY_LP(2) && results->wm_lp[1] != 0) + if (INTEL_INFO(dev)->gen >= 7) { + if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1]) + I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]); + if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2]) + I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]); + } + + if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0]) + I915_WRITE(WM1_LP_ILK, results->wm_lp[0]); + if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1]) I915_WRITE(WM2_LP_ILK, results->wm_lp[1]); - if (dirty & WM_DIRTY_LP(3) && results->wm_lp[2] != 0) + if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2]) I915_WRITE(WM3_LP_ILK, results->wm_lp[2]); dev_priv->wm.hw = *results; } -static void haswell_update_wm(struct drm_crtc *crtc) +static bool ilk_disable_lp_wm(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL); +} + +static void ilk_update_wm(struct drm_crtc *crtc) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct hsw_wm_maximums max; - struct hsw_pipe_wm_parameters params = {}; - struct hsw_wm_values results = {}; + struct ilk_wm_maximums max; + struct ilk_pipe_wm_parameters params = {}; + struct ilk_wm_values results = {}; enum intel_ddb_partitioning partitioning; struct intel_pipe_wm pipe_wm = {}; struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; struct intel_wm_config config = {}; - hsw_compute_wm_parameters(crtc, ¶ms, &config); + ilk_compute_wm_parameters(crtc, ¶ms, &config); intel_compute_pipe_wm(crtc, ¶ms, &pipe_wm); @@ -2945,15 +2555,15 @@ static void haswell_update_wm(struct drm_crtc *crtc) intel_crtc->wm.active = pipe_wm; ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max); - ilk_wm_merge(dev, &max, &lp_wm_1_2); + ilk_wm_merge(dev, &config, &max, &lp_wm_1_2); /* 5/6 split only in single pipe config on IVB+ */ if (INTEL_INFO(dev)->gen >= 7 && config.num_pipes_active == 1 && config.sprites_enabled) { ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max); - ilk_wm_merge(dev, &max, &lp_wm_5_6); + ilk_wm_merge(dev, &config, &max, &lp_wm_5_6); - best_lp_wm = hsw_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6); + best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6); } else { best_lp_wm = &lp_wm_1_2; } @@ -2961,16 +2571,17 @@ static void haswell_update_wm(struct drm_crtc *crtc) partitioning = (best_lp_wm == &lp_wm_1_2) ? INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6; - hsw_compute_wm_results(dev, best_lp_wm, partitioning, &results); + ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results); - hsw_write_wm_values(dev_priv, &results); + ilk_write_wm_values(dev_priv, &results); } -static void haswell_update_sprite_wm(struct drm_plane *plane, +static void ilk_update_sprite_wm(struct drm_plane *plane, struct drm_crtc *crtc, uint32_t sprite_width, int pixel_size, bool enabled, bool scaled) { + struct drm_device *dev = plane->dev; struct intel_plane *intel_plane = to_intel_plane(plane); intel_plane->wm.enabled = enabled; @@ -2978,176 +2589,24 @@ static void haswell_update_sprite_wm(struct drm_plane *plane, intel_plane->wm.horiz_pixels = sprite_width; intel_plane->wm.bytes_per_pixel = pixel_size; - haswell_update_wm(crtc); -} - -static bool -sandybridge_compute_sprite_wm(struct drm_device *dev, int plane, - uint32_t sprite_width, int pixel_size, - const struct intel_watermark_params *display, - int display_latency_ns, int *sprite_wm) -{ - struct drm_crtc *crtc; - int clock; - int entries, tlb_miss; - - crtc = intel_get_crtc_for_plane(dev, plane); - if (!intel_crtc_active(crtc)) { - *sprite_wm = display->guard_size; - return false; - } - - clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock; - - /* Use the small buffer method to calculate the sprite watermark */ - entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000; - tlb_miss = display->fifo_size*display->cacheline_size - - sprite_width * 8; - if (tlb_miss > 0) - entries += tlb_miss; - entries = DIV_ROUND_UP(entries, display->cacheline_size); - *sprite_wm = entries + display->guard_size; - if (*sprite_wm > (int)display->max_wm) - *sprite_wm = display->max_wm; - - return true; -} - -static bool -sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane, - uint32_t sprite_width, int pixel_size, - const struct intel_watermark_params *display, - int latency_ns, int *sprite_wm) -{ - struct drm_crtc *crtc; - unsigned long line_time_us; - int clock; - int line_count, line_size; - int small, large; - int entries; - - if (!latency_ns) { - *sprite_wm = 0; - return false; - } - - crtc = intel_get_crtc_for_plane(dev, plane); - clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock; - if (!clock) { - *sprite_wm = 0; - return false; - } - - line_time_us = (sprite_width * 1000) / clock; - if (!line_time_us) { - *sprite_wm = 0; - return false; - } - - line_count = (latency_ns / line_time_us + 1000) / 1000; - line_size = sprite_width * pixel_size; - - /* Use the minimum of the small and large buffer method for primary */ - small = ((clock * pixel_size / 1000) * latency_ns) / 1000; - large = line_count * line_size; - - entries = DIV_ROUND_UP(min(small, large), display->cacheline_size); - *sprite_wm = entries + display->guard_size; - - return *sprite_wm > 0x3ff ? false : true; -} - -static void sandybridge_update_sprite_wm(struct drm_plane *plane, - struct drm_crtc *crtc, - uint32_t sprite_width, int pixel_size, - bool enabled, bool scaled) -{ - struct drm_device *dev = plane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - int pipe = to_intel_plane(plane)->pipe; - int latency = dev_priv->wm.spr_latency[0] * 100; /* In unit 0.1us */ - u32 val; - int sprite_wm, reg; - int ret; - - if (!enabled) - return; - - switch (pipe) { - case 0: - reg = WM0_PIPEA_ILK; - break; - case 1: - reg = WM0_PIPEB_ILK; - break; - case 2: - reg = WM0_PIPEC_IVB; - break; - default: - return; /* bad pipe */ - } - - ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size, - &sandybridge_display_wm_info, - latency, &sprite_wm); - if (!ret) { - DRM_DEBUG_KMS("failed to compute sprite wm for pipe %c\n", - pipe_name(pipe)); - return; - } - - val = I915_READ(reg); - val &= ~WM0_PIPE_SPRITE_MASK; - I915_WRITE(reg, val | (sprite_wm << WM0_PIPE_SPRITE_SHIFT)); - DRM_DEBUG_KMS("sprite watermarks For pipe %c - %d\n", pipe_name(pipe), sprite_wm); - - - ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width, - pixel_size, - &sandybridge_display_srwm_info, - dev_priv->wm.spr_latency[1] * 500, - &sprite_wm); - if (!ret) { - DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %c\n", - pipe_name(pipe)); - return; - } - I915_WRITE(WM1S_LP_ILK, sprite_wm); - - /* Only IVB has two more LP watermarks for sprite */ - if (!IS_IVYBRIDGE(dev)) - return; - - ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width, - pixel_size, - &sandybridge_display_srwm_info, - dev_priv->wm.spr_latency[2] * 500, - &sprite_wm); - if (!ret) { - DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %c\n", - pipe_name(pipe)); - return; - } - I915_WRITE(WM2S_LP_IVB, sprite_wm); + /* + * IVB workaround: must disable low power watermarks for at least + * one frame before enabling scaling. LP watermarks can be re-enabled + * when scaling is disabled. + * + * WaCxSRDisabledForSpriteScaling:ivb + */ + if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev)) + intel_wait_for_vblank(dev, intel_plane->pipe); - ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width, - pixel_size, - &sandybridge_display_srwm_info, - dev_priv->wm.spr_latency[3] * 500, - &sprite_wm); - if (!ret) { - DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %c\n", - pipe_name(pipe)); - return; - } - I915_WRITE(WM3S_LP_IVB, sprite_wm); + ilk_update_wm(crtc); } static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct hsw_wm_values *hw = &dev_priv->wm.hw; + struct ilk_wm_values *hw = &dev_priv->wm.hw; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_pipe_wm *active = &intel_crtc->wm.active; enum pipe pipe = intel_crtc->pipe; @@ -3158,7 +2617,8 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) }; hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]); - hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe)); + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe)); if (intel_crtc_active(crtc)) { u32 tmp = hw->wm_pipe[pipe]; @@ -3190,7 +2650,7 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) void ilk_wm_get_hw_state(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct hsw_wm_values *hw = &dev_priv->wm.hw; + struct ilk_wm_values *hw = &dev_priv->wm.hw; struct drm_crtc *crtc; list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) @@ -3204,8 +2664,12 @@ void ilk_wm_get_hw_state(struct drm_device *dev) hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB); hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB); - hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ? - INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ? + INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; + else if (IS_IVYBRIDGE(dev)) + hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ? + INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; hw->enable_fbc_wm = !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS); @@ -3277,7 +2741,7 @@ intel_alloc_context_page(struct drm_device *dev) return NULL; } - ret = i915_gem_obj_ggtt_pin(ctx, 4096, true, false); + ret = i915_gem_obj_ggtt_pin(ctx, 4096, 0); if (ret) { DRM_ERROR("failed to pin power context: %d\n", ret); goto err_unref; @@ -3292,7 +2756,7 @@ intel_alloc_context_page(struct drm_device *dev) return ctx; err_unpin: - i915_gem_object_unpin(ctx); + i915_gem_object_ggtt_unpin(ctx); err_unref: drm_gem_object_unreference(&ctx->base); return NULL; @@ -3430,26 +2894,19 @@ static void ironlake_disable_drps(struct drm_device *dev) * ourselves, instead of doing a rmw cycle (which might result in us clearing * all limits and the gpu stuck at whatever frequency it is at atm). */ -static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 *val) +static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val) { u32 limits; - limits = 0; - - if (*val >= dev_priv->rps.max_delay) - *val = dev_priv->rps.max_delay; - limits |= dev_priv->rps.max_delay << 24; - /* Only set the down limit when we've reached the lowest level to avoid * getting more interrupts, otherwise leave this clear. This prevents a * race in the hw when coming out of rc6: There's a tiny window where * the hw runs at the minimal clock before selecting the desired * frequency, if the down threshold expires in that window we will not * receive a down interrupt. */ - if (*val <= dev_priv->rps.min_delay) { - *val = dev_priv->rps.min_delay; + limits = dev_priv->rps.max_delay << 24; + if (val <= dev_priv->rps.min_delay) limits |= dev_priv->rps.min_delay << 16; - } return limits; } @@ -3546,17 +3003,25 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) dev_priv->rps.last_adj = 0; } +/* gen6_set_rps is called to update the frequency request, but should also be + * called when the range (min_delay and max_delay) is modified so that we can + * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */ void gen6_set_rps(struct drm_device *dev, u8 val) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 limits = gen6_rps_limits(dev_priv, &val); WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); WARN_ON(val > dev_priv->rps.max_delay); WARN_ON(val < dev_priv->rps.min_delay); - if (val == dev_priv->rps.cur_delay) + if (val == dev_priv->rps.cur_delay) { + /* min/max delay may still have been modified so be sure to + * write the limits value */ + I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, + gen6_rps_limits(dev_priv, val)); + return; + } gen6_set_rps_thresholds(dev_priv, val); @@ -3572,7 +3037,8 @@ void gen6_set_rps(struct drm_device *dev, u8 val) /* Make sure we continue to get interrupts * until we hit the minimum or maximum frequencies. */ - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits); + I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, + gen6_rps_limits(dev_priv, val)); POSTING_READ(GEN6_RPNSWREQ); @@ -3581,12 +3047,66 @@ void gen6_set_rps(struct drm_device *dev, u8 val) trace_intel_gpu_freq_change(val * 50); } +/* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down + * + * * If Gfx is Idle, then + * 1. Mask Turbo interrupts + * 2. Bring up Gfx clock + * 3. Change the freq to Rpn and wait till P-Unit updates freq + * 4. Clear the Force GFX CLK ON bit so that Gfx can down + * 5. Unmask Turbo interrupts +*/ +static void vlv_set_rps_idle(struct drm_i915_private *dev_priv) +{ + /* + * When we are idle. Drop to min voltage state. + */ + + if (dev_priv->rps.cur_delay <= dev_priv->rps.min_delay) + return; + + /* Mask turbo interrupt so that they will not come in between */ + I915_WRITE(GEN6_PMINTRMSK, 0xffffffff); + + /* Bring up the Gfx clock */ + I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, + I915_READ(VLV_GTLC_SURVIVABILITY_REG) | + VLV_GFX_CLK_FORCE_ON_BIT); + + if (wait_for(((VLV_GFX_CLK_STATUS_BIT & + I915_READ(VLV_GTLC_SURVIVABILITY_REG)) != 0), 5)) { + DRM_ERROR("GFX_CLK_ON request timed out\n"); + return; + } + + dev_priv->rps.cur_delay = dev_priv->rps.min_delay; + + vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, + dev_priv->rps.min_delay); + + if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS)) + & GENFREQSTATUS) == 0, 5)) + DRM_ERROR("timed out waiting for Punit\n"); + + /* Release the Gfx clock */ + I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, + I915_READ(VLV_GTLC_SURVIVABILITY_REG) & + ~VLV_GFX_CLK_FORCE_ON_BIT); + + /* Unmask Up interrupts */ + dev_priv->rps.rp_up_masked = true; + gen6_set_pm_mask(dev_priv, GEN6_PM_RP_DOWN_THRESHOLD, + dev_priv->rps.min_delay); +} + void gen6_rps_idle(struct drm_i915_private *dev_priv) { + struct drm_device *dev = dev_priv->dev; + mutex_lock(&dev_priv->rps.hw_lock); if (dev_priv->rps.enabled) { - if (dev_priv->info->is_valleyview) - valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_delay); + if (IS_VALLEYVIEW(dev)) + vlv_set_rps_idle(dev_priv); else gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay); dev_priv->rps.last_adj = 0; @@ -3596,9 +3116,11 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv) void gen6_rps_boost(struct drm_i915_private *dev_priv) { + struct drm_device *dev = dev_priv->dev; + mutex_lock(&dev_priv->rps.hw_lock); if (dev_priv->rps.enabled) { - if (dev_priv->info->is_valleyview) + if (IS_VALLEYVIEW(dev)) valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_delay); else gen6_set_rps(dev_priv->dev, dev_priv->rps.max_delay); @@ -3607,48 +3129,18 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv) mutex_unlock(&dev_priv->rps.hw_lock); } -/* - * Wait until the previous freq change has completed, - * or the timeout elapsed, and then update our notion - * of the current GPU frequency. - */ -static void vlv_update_rps_cur_delay(struct drm_i915_private *dev_priv) -{ - u32 pval; - - WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); - - if (wait_for(((pval = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS)) & GENFREQSTATUS) == 0, 10)) - DRM_DEBUG_DRIVER("timed out waiting for Punit\n"); - - pval >>= 8; - - if (pval != dev_priv->rps.cur_delay) - DRM_DEBUG_DRIVER("Punit overrode GPU freq: %d MHz (%u) requested, but got %d Mhz (%u)\n", - vlv_gpu_freq(dev_priv->mem_freq, dev_priv->rps.cur_delay), - dev_priv->rps.cur_delay, - vlv_gpu_freq(dev_priv->mem_freq, pval), pval); - - dev_priv->rps.cur_delay = pval; -} - void valleyview_set_rps(struct drm_device *dev, u8 val) { struct drm_i915_private *dev_priv = dev->dev_private; - gen6_rps_limits(dev_priv, &val); - WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); WARN_ON(val > dev_priv->rps.max_delay); WARN_ON(val < dev_priv->rps.min_delay); - vlv_update_rps_cur_delay(dev_priv); - DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n", - vlv_gpu_freq(dev_priv->mem_freq, - dev_priv->rps.cur_delay), + vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay), dev_priv->rps.cur_delay, - vlv_gpu_freq(dev_priv->mem_freq, val), val); + vlv_gpu_freq(dev_priv, val), val); if (val == dev_priv->rps.cur_delay) return; @@ -3657,7 +3149,7 @@ void valleyview_set_rps(struct drm_device *dev, u8 val) dev_priv->rps.cur_delay = val; - trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv->mem_freq, val)); + trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv, val)); } static void gen6_disable_rps_interrupts(struct drm_device *dev) @@ -3704,16 +3196,10 @@ static void valleyview_disable_rps(struct drm_device *dev) static void intel_print_rc6_info(struct drm_device *dev, u32 mode) { - if (IS_GEN6(dev)) - DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n"); - - if (IS_HASWELL(dev)) - DRM_DEBUG_DRIVER("Haswell: only RC6 available\n"); - DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n", - (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off", - (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off", - (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off"); + (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off", + (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off", + (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off"); } int intel_enable_rc6(const struct drm_device *dev) @@ -3723,21 +3209,17 @@ int intel_enable_rc6(const struct drm_device *dev) return 0; /* Respect the kernel parameter if it is set */ - if (i915_enable_rc6 >= 0) - return i915_enable_rc6; + if (i915.enable_rc6 >= 0) + return i915.enable_rc6; /* Disable RC6 on Ironlake */ if (INTEL_INFO(dev)->gen == 5) return 0; - if (IS_HASWELL(dev)) - return INTEL_RC6_ENABLE; - - /* snb/ivb have more than one rc6 state. */ - if (INTEL_INFO(dev)->gen == 6) - return INTEL_RC6_ENABLE; + if (IS_IVYBRIDGE(dev)) + return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE); - return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE); + return INTEL_RC6_ENABLE; } static void gen6_enable_rps_interrupts(struct drm_device *dev) @@ -3775,7 +3257,7 @@ static void gen8_enable_rps(struct drm_device *dev) /* 1c & 1d: Get forcewake during program sequence. Although the driver * hasn't enabled a state yet where we need forcewake, BIOS may have.*/ - gen6_gt_force_wake_get(dev_priv); + gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); /* 2a: Disable RC states. */ I915_WRITE(GEN6_RC_CONTROL, 0); @@ -3794,10 +3276,10 @@ static void gen8_enable_rps(struct drm_device *dev) /* 3: Enable RC6 */ if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) rc6_mask = GEN6_RC_CTL_RC6_ENABLE; - DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off"); + intel_print_rc6_info(dev, rc6_mask); I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | - GEN6_RC_CTL_EI_MODE(1) | - rc6_mask); + GEN6_RC_CTL_EI_MODE(1) | + rc6_mask); /* 4 Program defaults and thresholds for RPS*/ I915_WRITE(GEN6_RPNSWREQ, HSW_FREQUENCY(10)); /* Request 500 MHz */ @@ -3832,14 +3314,14 @@ static void gen8_enable_rps(struct drm_device *dev) gen6_enable_rps_interrupts(dev); - gen6_gt_force_wake_put(dev_priv); + gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); } static void gen6_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_ring_buffer *ring; - u32 rp_state_cap; + u32 rp_state_cap, hw_max, hw_min; u32 gt_perf_status; u32 rc6vids, pcu_mbox, rc6_mask = 0; u32 gtfifodbg; @@ -3862,19 +3344,26 @@ static void gen6_enable_rps(struct drm_device *dev) I915_WRITE(GTFIFODBG, gtfifodbg); } - gen6_gt_force_wake_get(dev_priv); + gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS); /* In units of 50MHz */ - dev_priv->rps.hw_max = dev_priv->rps.max_delay = rp_state_cap & 0xff; - dev_priv->rps.min_delay = (rp_state_cap >> 16) & 0xff; + dev_priv->rps.hw_max = hw_max = rp_state_cap & 0xff; + hw_min = (rp_state_cap >> 16) & 0xff; dev_priv->rps.rp1_delay = (rp_state_cap >> 8) & 0xff; dev_priv->rps.rp0_delay = (rp_state_cap >> 0) & 0xff; dev_priv->rps.rpe_delay = dev_priv->rps.rp1_delay; dev_priv->rps.cur_delay = 0; + /* Preserve min/max settings in case of re-init */ + if (dev_priv->rps.max_delay == 0) + dev_priv->rps.max_delay = hw_max; + + if (dev_priv->rps.min_delay == 0) + dev_priv->rps.min_delay = hw_min; + /* disable the counters and set deterministic thresholds */ I915_WRITE(GEN6_RC_CONTROL, 0); @@ -3954,7 +3443,7 @@ static void gen6_enable_rps(struct drm_device *dev) DRM_ERROR("Couldn't fix incorrect rc6 voltage\n"); } - gen6_gt_force_wake_put(dev_priv); + gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); } void gen6_update_ring_freq(struct drm_device *dev) @@ -4103,7 +3592,7 @@ static void valleyview_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_ring_buffer *ring; - u32 gtfifodbg, val, rc6_mode = 0; + u32 gtfifodbg, val, hw_max, hw_min, rc6_mode = 0; int i; WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); @@ -4116,7 +3605,8 @@ static void valleyview_enable_rps(struct drm_device *dev) valleyview_setup_pctx(dev); - gen6_gt_force_wake_get(dev_priv); + /* If VLV, Forcewake all wells, else re-direct to regular path */ + gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400); I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000); @@ -4140,7 +3630,7 @@ static void valleyview_enable_rps(struct drm_device *dev) for_each_ring(ring, dev_priv, i) I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); - I915_WRITE(GEN6_RC6_THRESHOLD, 0xc350); + I915_WRITE(GEN6_RC6_THRESHOLD, 0x557); /* allows RC6 residency counter to work */ I915_WRITE(VLV_COUNTER_CONTROL, @@ -4148,65 +3638,56 @@ static void valleyview_enable_rps(struct drm_device *dev) VLV_MEDIA_RC6_COUNT_EN | VLV_RENDER_RC6_COUNT_EN)); if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) - rc6_mode = GEN7_RC_CTL_TO_MODE; + rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL; intel_print_rc6_info(dev, rc6_mode); I915_WRITE(GEN6_RC_CONTROL, rc6_mode); val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); - switch ((val >> 6) & 3) { - case 0: - case 1: - dev_priv->mem_freq = 800; - break; - case 2: - dev_priv->mem_freq = 1066; - break; - case 3: - dev_priv->mem_freq = 1333; - break; - } - DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no"); DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); dev_priv->rps.cur_delay = (val >> 8) & 0xff; DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n", - vlv_gpu_freq(dev_priv->mem_freq, - dev_priv->rps.cur_delay), + vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay), dev_priv->rps.cur_delay); - dev_priv->rps.max_delay = valleyview_rps_max_freq(dev_priv); - dev_priv->rps.hw_max = dev_priv->rps.max_delay; + dev_priv->rps.hw_max = hw_max = valleyview_rps_max_freq(dev_priv); DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", - vlv_gpu_freq(dev_priv->mem_freq, - dev_priv->rps.max_delay), - dev_priv->rps.max_delay); + vlv_gpu_freq(dev_priv, hw_max), + hw_max); dev_priv->rps.rpe_delay = valleyview_rps_rpe_freq(dev_priv); DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", - vlv_gpu_freq(dev_priv->mem_freq, - dev_priv->rps.rpe_delay), + vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay), dev_priv->rps.rpe_delay); - dev_priv->rps.min_delay = valleyview_rps_min_freq(dev_priv); + hw_min = valleyview_rps_min_freq(dev_priv); DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", - vlv_gpu_freq(dev_priv->mem_freq, - dev_priv->rps.min_delay), - dev_priv->rps.min_delay); + vlv_gpu_freq(dev_priv, hw_min), + hw_min); + + /* Preserve min/max settings in case of re-init */ + if (dev_priv->rps.max_delay == 0) + dev_priv->rps.max_delay = hw_max; + + if (dev_priv->rps.min_delay == 0) + dev_priv->rps.min_delay = hw_min; DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n", - vlv_gpu_freq(dev_priv->mem_freq, - dev_priv->rps.rpe_delay), + vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay), dev_priv->rps.rpe_delay); valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay); + dev_priv->rps.rp_up_masked = false; + dev_priv->rps.rp_down_masked = false; + gen6_enable_rps_interrupts(dev); - gen6_gt_force_wake_put(dev_priv); + gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); } void ironlake_teardown_rc6(struct drm_device *dev) @@ -4214,13 +3695,13 @@ void ironlake_teardown_rc6(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; if (dev_priv->ips.renderctx) { - i915_gem_object_unpin(dev_priv->ips.renderctx); + i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx); drm_gem_object_unreference(&dev_priv->ips.renderctx->base); dev_priv->ips.renderctx = NULL; } if (dev_priv->ips.pwrctx) { - i915_gem_object_unpin(dev_priv->ips.pwrctx); + i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx); drm_gem_object_unreference(&dev_priv->ips.pwrctx->base); dev_priv->ips.pwrctx = NULL; } @@ -4412,9 +3893,10 @@ static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv) unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) { + struct drm_device *dev = dev_priv->dev; unsigned long val; - if (dev_priv->info->gen != 5) + if (INTEL_INFO(dev)->gen != 5) return 0; spin_lock_irq(&mchdev_lock); @@ -4443,6 +3925,7 @@ unsigned long i915_mch_val(struct drm_i915_private *dev_priv) static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) { + struct drm_device *dev = dev_priv->dev; static const struct v_table { u16 vd; /* in .1 mil */ u16 vm; /* in .1 mil */ @@ -4576,7 +4059,7 @@ static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) { 16000, 14875, }, { 16125, 15000, }, }; - if (dev_priv->info->is_mobile) + if (INTEL_INFO(dev)->is_mobile) return v_table[pxvid].vm; else return v_table[pxvid].vd; @@ -4619,7 +4102,9 @@ static void __i915_update_gfx_val(struct drm_i915_private *dev_priv) void i915_update_gfx_val(struct drm_i915_private *dev_priv) { - if (dev_priv->info->gen != 5) + struct drm_device *dev = dev_priv->dev; + + if (INTEL_INFO(dev)->gen != 5) return; spin_lock_irq(&mchdev_lock); @@ -4668,9 +4153,10 @@ static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) { + struct drm_device *dev = dev_priv->dev; unsigned long val; - if (dev_priv->info->gen != 5) + if (INTEL_INFO(dev)->gen != 5) return 0; spin_lock_irq(&mchdev_lock); @@ -4859,6 +4345,7 @@ void intel_gpu_ips_teardown(void) i915_mch_dev = NULL; spin_unlock_irq(&mchdev_lock); } + static void intel_init_emon(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -5019,6 +4506,20 @@ static void g4x_disable_trickle_feed(struct drm_device *dev) } } +static void ilk_init_lp_watermarks(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN); + I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN); + I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN); + + /* + * Don't touch WM1S_LP_EN here. + * Doing so could cause underruns. + */ +} + static void ironlake_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -5052,9 +4553,8 @@ static void ironlake_init_clock_gating(struct drm_device *dev) I915_WRITE(DISP_ARB_CTL, (I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS)); - I915_WRITE(WM3_LP_ILK, 0); - I915_WRITE(WM2_LP_ILK, 0); - I915_WRITE(WM1_LP_ILK, 0); + + ilk_init_lp_watermarks(dev); /* * Based on the document from hardware guys the following bits @@ -5161,9 +4661,18 @@ static void gen6_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE)); - I915_WRITE(WM3_LP_ILK, 0); - I915_WRITE(WM2_LP_ILK, 0); - I915_WRITE(WM1_LP_ILK, 0); + /* + * BSpec recoomends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + I915_WRITE(GEN6_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); + + ilk_init_lp_watermarks(dev); I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); @@ -5183,17 +4692,24 @@ static void gen6_init_clock_gating(struct drm_device *dev) * According to the spec, bit 11 (RCCUNIT) must also be set, * but we didn't debug actual testcases to find it out. * - * Also apply WaDisableVDSUnitClockGating:snb and - * WaDisableRCPBUnitClockGating:snb. + * WaDisableRCCUnitClockGating:snb + * WaDisableRCPBUnitClockGating:snb */ I915_WRITE(GEN6_UCGCTL2, - GEN7_VDSUNIT_CLOCK_GATE_DISABLE | GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | GEN6_RCCUNIT_CLOCK_GATE_DISABLE); - /* Bspec says we need to always set all mask bits. */ - I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) | - _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL); + /* WaStripsFansDisableFastClipPerformanceFix:snb */ + I915_WRITE(_3D_CHICKEN3, + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL)); + + /* + * Bspec says: + * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and + * 3DSTATE_SF number of SF output attributes is more than 16." + */ + I915_WRITE(_3D_CHICKEN3, + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH)); /* * According to the spec the following bits should be @@ -5219,11 +4735,6 @@ static void gen6_init_clock_gating(struct drm_device *dev) g4x_disable_trickle_feed(dev); - /* The default value should be 0x200 according to docs, but the two - * platforms I checked have a 0 for this. (Maybe BIOS overrides?) */ - I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff)); - I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI)); - cpt_init_clock_gating(dev); gen6_check_mch_setup(dev); @@ -5233,14 +4744,17 @@ static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) { uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE); + /* + * WaVSThreadDispatchOverride:ivb,vlv + * + * This actually overrides the dispatch + * mode for all thread types. + */ reg &= ~GEN7_FF_SCHED_MASK; reg |= GEN7_FF_TS_SCHED_HW; reg |= GEN7_FF_VS_SCHED_HW; reg |= GEN7_FF_DS_SCHED_HW; - if (IS_HASWELL(dev_priv->dev)) - reg &= ~GEN7_FF_VS_REF_CNT_FFME; - I915_WRITE(GEN7_FF_THREAD_MODE, reg); } @@ -5287,8 +4801,10 @@ static void gen8_init_clock_gating(struct drm_device *dev) /* FIXME(BDW): Check all the w/a, some might only apply to * pre-production hw. */ - WARN(!i915_preliminary_hw_support, - "GEN8_CENTROID_PIXEL_OPT_DIS not be needed for production\n"); + /* + * This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for + * pre-production hardware + */ I915_WRITE(HALF_SLICE_CHICKEN3, _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS)); I915_WRITE(HALF_SLICE_CHICKEN3, @@ -5304,43 +4820,51 @@ static void gen8_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, _MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE)); - /* WaSwitchSolVfFArbitrationPriority */ + /* WaSwitchSolVfFArbitrationPriority:bdw */ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); - /* WaPsrDPAMaskVBlankInSRD */ + /* WaPsrDPAMaskVBlankInSRD:bdw */ I915_WRITE(CHICKEN_PAR1_1, I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD); - /* WaPsrDPRSUnmaskVBlankInSRD */ + /* WaPsrDPRSUnmaskVBlankInSRD:bdw */ for_each_pipe(i) { I915_WRITE(CHICKEN_PIPESL_1(i), I915_READ(CHICKEN_PIPESL_1(i) | DPRS_MASK_VBLANK_SRD)); } -} -static void haswell_init_clock_gating(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; + /* Use Force Non-Coherent whenever executing a 3D context. This is a + * workaround for for a possible hang in the unlikely event a TLB + * invalidation occurs during a PSD flush. + */ + I915_WRITE(HDC_CHICKEN0, + I915_READ(HDC_CHICKEN0) | + _MASKED_BIT_ENABLE(HDC_FORCE_NON_COHERENT)); - I915_WRITE(WM3_LP_ILK, 0); - I915_WRITE(WM2_LP_ILK, 0); - I915_WRITE(WM1_LP_ILK, 0); + /* WaVSRefCountFullforceMissDisable:bdw */ + /* WaDSRefCountFullforceMissDisable:bdw */ + I915_WRITE(GEN7_FF_THREAD_MODE, + I915_READ(GEN7_FF_THREAD_MODE) & + ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); - /* According to the spec, bit 13 (RCZUNIT) must be set on IVB. - * This implements the WaDisableRCZUnitClockGating:hsw workaround. + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). */ - I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE); + I915_WRITE(GEN7_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); +} - /* Apply the WaDisableRHWOOptimizationForRenderHang:hsw workaround. */ - I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, - GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); +static void haswell_init_clock_gating(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; - /* WaApplyL3ControlAndL3ChickenMode:hsw */ - I915_WRITE(GEN7_L3CNTLREG1, - GEN7_WA_FOR_GEN7_L3_CONTROL); - I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, - GEN7_WA_L3_CHICKEN_MODE); + ilk_init_lp_watermarks(dev); /* L3 caching of data atomics doesn't work -- disable it. */ I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); @@ -5353,12 +4877,28 @@ static void haswell_init_clock_gating(struct drm_device *dev) GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); /* WaVSRefCountFullforceMissDisable:hsw */ - gen7_setup_fixed_func_scheduler(dev_priv); + I915_WRITE(GEN7_FF_THREAD_MODE, + I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME); + + /* enable HiZ Raw Stall Optimization */ + I915_WRITE(CACHE_MODE_0_GEN7, + _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); /* WaDisable4x2SubspanOptimization:hsw */ I915_WRITE(CACHE_MODE_1, _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + I915_WRITE(GEN7_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); + /* WaSwitchSolVfFArbitrationPriority:hsw */ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); @@ -5374,9 +4914,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; uint32_t snpcr; - I915_WRITE(WM3_LP_ILK, 0); - I915_WRITE(WM2_LP_ILK, 0); - I915_WRITE(WM1_LP_ILK, 0); + ilk_init_lp_watermarks(dev); I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE); @@ -5393,9 +4931,6 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) if (IS_IVB_GT1(dev)) I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); - else - I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2, - _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */ I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, @@ -5409,31 +4944,24 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) if (IS_IVB_GT1(dev)) I915_WRITE(GEN7_ROW_CHICKEN2, _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); - else + else { + /* must write both registers */ + I915_WRITE(GEN7_ROW_CHICKEN2, + _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); I915_WRITE(GEN7_ROW_CHICKEN2_GT2, _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); - + } /* WaForceL3Serialization:ivb */ I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & ~L3SQ_URB_READ_CAM_MATCH_DISABLE); - /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock - * gating disable must be set. Failure to set it results in - * flickering pixels due to Z write ordering failures after - * some amount of runtime in the Mesa "fire" demo, and Unigine - * Sanctuary and Tropics, and apparently anything else with - * alpha test or pixel discard. - * - * According to the spec, bit 11 (RCCUNIT) must also be set, - * but we didn't debug actual testcases to find it out. - * + /* * According to the spec, bit 13 (RCZUNIT) must be set on IVB. * This implements the WaDisableRCZUnitClockGating:ivb workaround. */ I915_WRITE(GEN6_UCGCTL2, - GEN6_RCZUNIT_CLOCK_GATE_DISABLE | - GEN6_RCCUNIT_CLOCK_GATE_DISABLE); + GEN6_RCZUNIT_CLOCK_GATE_DISABLE); /* This is required by WaCatErrorRejectionIssue:ivb */ I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, @@ -5442,13 +4970,29 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) g4x_disable_trickle_feed(dev); - /* WaVSRefCountFullforceMissDisable:ivb */ gen7_setup_fixed_func_scheduler(dev_priv); + if (0) { /* causes HiZ corruption on ivb:gt1 */ + /* enable HiZ Raw Stall Optimization */ + I915_WRITE(CACHE_MODE_0_GEN7, + _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); + } + /* WaDisable4x2SubspanOptimization:ivb */ I915_WRITE(CACHE_MODE_1, _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + I915_WRITE(GEN7_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); + snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); snpcr &= ~GEN6_MBC_SNPCR_MASK; snpcr |= GEN6_MBC_SNPCR_MED; @@ -5463,6 +5007,26 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) static void valleyview_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + u32 val; + + mutex_lock(&dev_priv->rps.hw_lock); + val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); + mutex_unlock(&dev_priv->rps.hw_lock); + switch ((val >> 6) & 3) { + case 0: + dev_priv->mem_freq = 800; + break; + case 1: + dev_priv->mem_freq = 1066; + break; + case 2: + dev_priv->mem_freq = 1333; + break; + case 3: + dev_priv->mem_freq = 1333; + break; + } + DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE); @@ -5475,19 +5039,12 @@ static void valleyview_init_clock_gating(struct drm_device *dev) CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | CHICKEN3_DGMG_DONE_FIX_DISABLE); + /* WaPsdDispatchEnable:vlv */ /* WaDisablePSDDualDispatchEnable:vlv */ I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP | GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); - /* Apply the WaDisableRHWOOptimizationForRenderHang:vlv workaround. */ - I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, - GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); - - /* WaApplyL3ControlAndL3ChickenMode:vlv */ - I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) | GEN7_L3AGDIS); - I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE); - /* WaForceL3Serialization:vlv */ I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & ~L3SQ_URB_READ_CAM_MATCH_DISABLE); @@ -5501,51 +5058,39 @@ static void valleyview_init_clock_gating(struct drm_device *dev) I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); - /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock - * gating disable must be set. Failure to set it results in - * flickering pixels due to Z write ordering failures after - * some amount of runtime in the Mesa "fire" demo, and Unigine - * Sanctuary and Tropics, and apparently anything else with - * alpha test or pixel discard. - * - * According to the spec, bit 11 (RCCUNIT) must also be set, - * but we didn't debug actual testcases to find it out. - * + gen7_setup_fixed_func_scheduler(dev_priv); + + /* * According to the spec, bit 13 (RCZUNIT) must be set on IVB. * This implements the WaDisableRCZUnitClockGating:vlv workaround. - * - * Also apply WaDisableVDSUnitClockGating:vlv and - * WaDisableRCPBUnitClockGating:vlv. */ I915_WRITE(GEN6_UCGCTL2, - GEN7_VDSUNIT_CLOCK_GATE_DISABLE | - GEN7_TDLUNIT_CLOCK_GATE_DISABLE | - GEN6_RCZUNIT_CLOCK_GATE_DISABLE | - GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | - GEN6_RCCUNIT_CLOCK_GATE_DISABLE); + GEN6_RCZUNIT_CLOCK_GATE_DISABLE); + /* WaDisableL3Bank2xClockGate:vlv */ I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE); I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE); + /* + * BSpec says this must be set, even though + * WaDisable4x2SubspanOptimization isn't listed for VLV. + */ I915_WRITE(CACHE_MODE_1, _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); + /* + * WaIncreaseL3CreditsForVLVB0:vlv + * This is the hardware default actually. + */ + I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE); + /* * WaDisableVLVClockGating_VBIIssue:vlv * Disable clock gating on th GCFG unit to prevent a delay * in the reporting of vblank events. */ - I915_WRITE(VLV_GUNIT_CLOCK_GATE, 0xffffffff); - - /* Conservative clock gating settings for now */ - I915_WRITE(0x9400, 0xffffffff); - I915_WRITE(0x9404, 0xffffffff); - I915_WRITE(0x9408, 0xffffffff); - I915_WRITE(0x940c, 0xffffffff); - I915_WRITE(0x9410, 0xffffffff); - I915_WRITE(0x9414, 0xffffffff); - I915_WRITE(0x9418, 0xffffffff); + I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS); } static void g4x_init_clock_gating(struct drm_device *dev) @@ -5642,52 +5187,132 @@ void intel_suspend_hw(struct drm_device *dev) lpt_suspend_hw(dev); } -static bool is_always_on_power_domain(struct drm_device *dev, - enum intel_display_power_domain domain) -{ - unsigned long always_on_domains; - - BUG_ON(BIT(domain) & ~POWER_DOMAIN_MASK); - - if (IS_BROADWELL(dev)) { - always_on_domains = BDW_ALWAYS_ON_POWER_DOMAINS; - } else if (IS_HASWELL(dev)) { - always_on_domains = HSW_ALWAYS_ON_POWER_DOMAINS; - } else { - WARN_ON(1); - return true; - } +#define for_each_power_well(i, power_well, domain_mask, power_domains) \ + for (i = 0; \ + i < (power_domains)->power_well_count && \ + ((power_well) = &(power_domains)->power_wells[i]); \ + i++) \ + if ((power_well)->domains & (domain_mask)) - return BIT(domain) & always_on_domains; -} +#define for_each_power_well_rev(i, power_well, domain_mask, power_domains) \ + for (i = (power_domains)->power_well_count - 1; \ + i >= 0 && ((power_well) = &(power_domains)->power_wells[i]);\ + i--) \ + if ((power_well)->domains & (domain_mask)) /** * We should only use the power well if we explicitly asked the hardware to * enable it, so check if it's enabled and also check if we've requested it to * be enabled. */ -bool intel_display_power_enabled(struct drm_device *dev, +static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + return I915_READ(HSW_PWR_WELL_DRIVER) == + (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED); +} + +bool intel_display_power_enabled_sw(struct drm_i915_private *dev_priv, + enum intel_display_power_domain domain) +{ + struct i915_power_domains *power_domains; + + power_domains = &dev_priv->power_domains; + + return power_domains->domain_use_count[domain]; +} + +bool intel_display_power_enabled(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct i915_power_domains *power_domains; + struct i915_power_well *power_well; + bool is_enabled; + int i; - if (!HAS_POWER_WELL(dev)) - return true; + power_domains = &dev_priv->power_domains; - if (is_always_on_power_domain(dev, domain)) - return true; + is_enabled = true; - return I915_READ(HSW_PWR_WELL_DRIVER) == - (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED); + mutex_lock(&power_domains->lock); + for_each_power_well_rev(i, power_well, BIT(domain), power_domains) { + if (power_well->always_on) + continue; + + if (!power_well->is_enabled(dev_priv, power_well)) { + is_enabled = false; + break; + } + } + mutex_unlock(&power_domains->lock); + + return is_enabled; } -static void __intel_set_power_well(struct drm_device *dev, bool enable) +static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - bool is_enabled, enable_requested; + struct drm_device *dev = dev_priv->dev; + unsigned long irqflags; + + /* + * After we re-enable the power well, if we touch VGA register 0x3d5 + * we'll get unclaimed register interrupts. This stops after we write + * anything to the VGA MSR register. The vgacon module uses this + * register all the time, so if we unbind our driver and, as a + * consequence, bind vgacon, we'll get stuck in an infinite loop at + * console_unlock(). So make here we touch the VGA MSR register, making + * sure vgacon can keep working normally without triggering interrupts + * and error messages. + */ + vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO); + outb(inb(VGA_MSR_READ), VGA_MSR_WRITE); + vga_put(dev->pdev, VGA_RSRC_LEGACY_IO); + + if (IS_BROADWELL(dev)) { + spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_B), + dev_priv->de_irq_mask[PIPE_B]); + I915_WRITE(GEN8_DE_PIPE_IER(PIPE_B), + ~dev_priv->de_irq_mask[PIPE_B] | + GEN8_PIPE_VBLANK); + I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_C), + dev_priv->de_irq_mask[PIPE_C]); + I915_WRITE(GEN8_DE_PIPE_IER(PIPE_C), + ~dev_priv->de_irq_mask[PIPE_C] | + GEN8_PIPE_VBLANK); + POSTING_READ(GEN8_DE_PIPE_IER(PIPE_C)); + spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + } +} + +static void hsw_power_well_post_disable(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + enum pipe p; unsigned long irqflags; + + /* + * After this, the registers on the pipes that are part of the power + * well will become zero, so we have to adjust our counters according to + * that. + * + * FIXME: Should we do this in general in drm_vblank_post_modeset? + */ + spin_lock_irqsave(&dev->vbl_lock, irqflags); + for_each_pipe(p) + if (p != PIPE_A) + dev->vblank[p].last = 0; + spin_unlock_irqrestore(&dev->vbl_lock, irqflags); +} + +static void hsw_set_power_well(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well, bool enable) +{ + bool is_enabled, enable_requested; uint32_t tmp; + WARN_ON(dev_priv->pc8.enabled); + tmp = I915_READ(HSW_PWR_WELL_DRIVER); is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED; enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST; @@ -5704,96 +5329,75 @@ static void __intel_set_power_well(struct drm_device *dev, bool enable) DRM_ERROR("Timeout enabling power well\n"); } - if (IS_BROADWELL(dev)) { - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_B), - dev_priv->de_irq_mask[PIPE_B]); - I915_WRITE(GEN8_DE_PIPE_IER(PIPE_B), - ~dev_priv->de_irq_mask[PIPE_B] | - GEN8_PIPE_VBLANK); - I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_C), - dev_priv->de_irq_mask[PIPE_C]); - I915_WRITE(GEN8_DE_PIPE_IER(PIPE_C), - ~dev_priv->de_irq_mask[PIPE_C] | - GEN8_PIPE_VBLANK); - POSTING_READ(GEN8_DE_PIPE_IER(PIPE_C)); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); - } + hsw_power_well_post_enable(dev_priv); } else { if (enable_requested) { - enum pipe p; - I915_WRITE(HSW_PWR_WELL_DRIVER, 0); POSTING_READ(HSW_PWR_WELL_DRIVER); DRM_DEBUG_KMS("Requesting to disable the power well\n"); - /* - * After this, the registers on the pipes that are part - * of the power well will become zero, so we have to - * adjust our counters according to that. - * - * FIXME: Should we do this in general in - * drm_vblank_post_modeset? - */ - spin_lock_irqsave(&dev->vbl_lock, irqflags); - for_each_pipe(p) - if (p != PIPE_A) - dev->vblank[p].last = 0; - spin_unlock_irqrestore(&dev->vbl_lock, irqflags); + hsw_power_well_post_disable(dev_priv); } } } -static void __intel_power_well_get(struct drm_device *dev, +static void __intel_power_well_get(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - if (!power_well->count++) - __intel_set_power_well(dev, true); + if (!power_well->count++ && power_well->set) { + hsw_disable_package_c8(dev_priv); + power_well->set(dev_priv, power_well, true); + } } -static void __intel_power_well_put(struct drm_device *dev, +static void __intel_power_well_put(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { WARN_ON(!power_well->count); - if (!--power_well->count && i915_disable_power_well) - __intel_set_power_well(dev, false); + + if (!--power_well->count && power_well->set && + i915.disable_power_well) { + power_well->set(dev_priv, power_well, false); + hsw_enable_package_c8(dev_priv); + } } -void intel_display_power_get(struct drm_device *dev, +void intel_display_power_get(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains; - - if (!HAS_POWER_WELL(dev)) - return; - - if (is_always_on_power_domain(dev, domain)) - return; + struct i915_power_well *power_well; + int i; power_domains = &dev_priv->power_domains; mutex_lock(&power_domains->lock); - __intel_power_well_get(dev, &power_domains->power_wells[0]); + + for_each_power_well(i, power_well, BIT(domain), power_domains) + __intel_power_well_get(dev_priv, power_well); + + power_domains->domain_use_count[domain]++; + mutex_unlock(&power_domains->lock); } -void intel_display_power_put(struct drm_device *dev, +void intel_display_power_put(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains; - - if (!HAS_POWER_WELL(dev)) - return; - - if (is_always_on_power_domain(dev, domain)) - return; + struct i915_power_well *power_well; + int i; power_domains = &dev_priv->power_domains; mutex_lock(&power_domains->lock); - __intel_power_well_put(dev, &power_domains->power_wells[0]); + + WARN_ON(!power_domains->domain_use_count[domain]); + power_domains->domain_use_count[domain]--; + + for_each_power_well_rev(i, power_well, BIT(domain), power_domains) + __intel_power_well_put(dev_priv, power_well); + mutex_unlock(&power_domains->lock); } @@ -5809,10 +5413,7 @@ void i915_request_power_well(void) dev_priv = container_of(hsw_pwr, struct drm_i915_private, power_domains); - - mutex_lock(&hsw_pwr->lock); - __intel_power_well_get(dev_priv->dev, &hsw_pwr->power_wells[0]); - mutex_unlock(&hsw_pwr->lock); + intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO); } EXPORT_SYMBOL_GPL(i915_request_power_well); @@ -5826,47 +5427,90 @@ void i915_release_power_well(void) dev_priv = container_of(hsw_pwr, struct drm_i915_private, power_domains); - - mutex_lock(&hsw_pwr->lock); - __intel_power_well_put(dev_priv->dev, &hsw_pwr->power_wells[0]); - mutex_unlock(&hsw_pwr->lock); + intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO); } EXPORT_SYMBOL_GPL(i915_release_power_well); -int intel_power_domains_init(struct drm_device *dev) +static struct i915_power_well i9xx_always_on_power_well[] = { + { + .name = "always-on", + .always_on = 1, + .domains = POWER_DOMAIN_MASK, + }, +}; + +static struct i915_power_well hsw_power_wells[] = { + { + .name = "always-on", + .always_on = 1, + .domains = HSW_ALWAYS_ON_POWER_DOMAINS, + }, + { + .name = "display", + .domains = POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS, + .is_enabled = hsw_power_well_enabled, + .set = hsw_set_power_well, + }, +}; + +static struct i915_power_well bdw_power_wells[] = { + { + .name = "always-on", + .always_on = 1, + .domains = BDW_ALWAYS_ON_POWER_DOMAINS, + }, + { + .name = "display", + .domains = POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS, + .is_enabled = hsw_power_well_enabled, + .set = hsw_set_power_well, + }, +}; + +#define set_power_wells(power_domains, __power_wells) ({ \ + (power_domains)->power_wells = (__power_wells); \ + (power_domains)->power_well_count = ARRAY_SIZE(__power_wells); \ +}) + +int intel_power_domains_init(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains = &dev_priv->power_domains; - struct i915_power_well *power_well; mutex_init(&power_domains->lock); - hsw_pwr = power_domains; - power_well = &power_domains->power_wells[0]; - power_well->count = 0; + /* + * The enabling order will be from lower to higher indexed wells, + * the disabling order is reversed. + */ + if (IS_HASWELL(dev_priv->dev)) { + set_power_wells(power_domains, hsw_power_wells); + hsw_pwr = power_domains; + } else if (IS_BROADWELL(dev_priv->dev)) { + set_power_wells(power_domains, bdw_power_wells); + hsw_pwr = power_domains; + } else { + set_power_wells(power_domains, i9xx_always_on_power_well); + } return 0; } -void intel_power_domains_remove(struct drm_device *dev) +void intel_power_domains_remove(struct drm_i915_private *dev_priv) { hsw_pwr = NULL; } -static void intel_power_domains_resume(struct drm_device *dev) +static void intel_power_domains_resume(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains = &dev_priv->power_domains; struct i915_power_well *power_well; - - if (!HAS_POWER_WELL(dev)) - return; + int i; mutex_lock(&power_domains->lock); - - power_well = &power_domains->power_wells[0]; - __intel_set_power_well(dev, power_well->count > 0); - + for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) { + if (power_well->set) + power_well->set(dev_priv, power_well, power_well->count > 0); + } mutex_unlock(&power_domains->lock); } @@ -5876,17 +5520,15 @@ static void intel_power_domains_resume(struct drm_device *dev) * to be enabled, and it will only be disabled if none of the registers is * requesting it to be enabled. */ -void intel_power_domains_init_hw(struct drm_device *dev) +void intel_power_domains_init_hw(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; + /* For now, we need the power well to be always enabled. */ + intel_display_set_init_power(dev_priv, true); + intel_power_domains_resume(dev_priv); - if (!HAS_POWER_WELL(dev)) + if (!(IS_HASWELL(dev_priv->dev) || IS_BROADWELL(dev_priv->dev))) return; - /* For now, we need the power well to be always enabled. */ - intel_display_set_init_power(dev, true); - intel_power_domains_resume(dev); - /* We're taking over the BIOS, so clear any requests made by it since * the driver is in charge now. */ if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST) @@ -5904,31 +5546,86 @@ void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv) hsw_enable_package_c8(dev_priv); } +void intel_runtime_pm_get(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + struct device *device = &dev->pdev->dev; + + if (!HAS_RUNTIME_PM(dev)) + return; + + pm_runtime_get_sync(device); + WARN(dev_priv->pm.suspended, "Device still suspended.\n"); +} + +void intel_runtime_pm_put(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + struct device *device = &dev->pdev->dev; + + if (!HAS_RUNTIME_PM(dev)) + return; + + pm_runtime_mark_last_busy(device); + pm_runtime_put_autosuspend(device); +} + +void intel_init_runtime_pm(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + struct device *device = &dev->pdev->dev; + + dev_priv->pm.suspended = false; + + if (!HAS_RUNTIME_PM(dev)) + return; + + pm_runtime_set_active(device); + + pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */ + pm_runtime_mark_last_busy(device); + pm_runtime_use_autosuspend(device); +} + +void intel_fini_runtime_pm(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + struct device *device = &dev->pdev->dev; + + if (!HAS_RUNTIME_PM(dev)) + return; + + /* Make sure we're not suspended first. */ + pm_runtime_get_sync(device); + pm_runtime_disable(device); +} + /* Set up chip specific power management-related functions */ void intel_init_pm(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - if (I915_HAS_FBC(dev)) { - if (HAS_PCH_SPLIT(dev)) { + if (HAS_FBC(dev)) { + if (INTEL_INFO(dev)->gen >= 7) { dev_priv->display.fbc_enabled = ironlake_fbc_enabled; - if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) - dev_priv->display.enable_fbc = - gen7_enable_fbc; - else - dev_priv->display.enable_fbc = - ironlake_enable_fbc; + dev_priv->display.enable_fbc = gen7_enable_fbc; + dev_priv->display.disable_fbc = ironlake_disable_fbc; + } else if (INTEL_INFO(dev)->gen >= 5) { + dev_priv->display.fbc_enabled = ironlake_fbc_enabled; + dev_priv->display.enable_fbc = ironlake_enable_fbc; dev_priv->display.disable_fbc = ironlake_disable_fbc; } else if (IS_GM45(dev)) { dev_priv->display.fbc_enabled = g4x_fbc_enabled; dev_priv->display.enable_fbc = g4x_enable_fbc; dev_priv->display.disable_fbc = g4x_disable_fbc; - } else if (IS_CRESTLINE(dev)) { + } else { dev_priv->display.fbc_enabled = i8xx_fbc_enabled; dev_priv->display.enable_fbc = i8xx_enable_fbc; dev_priv->display.disable_fbc = i8xx_disable_fbc; + + /* This value was pulled out of someone's hat */ + I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT); } - /* 855GM needs testing */ } /* For cxsr */ @@ -5941,58 +5638,27 @@ void intel_init_pm(struct drm_device *dev) if (HAS_PCH_SPLIT(dev)) { intel_setup_wm_latency(dev); - if (IS_GEN5(dev)) { - if (dev_priv->wm.pri_latency[1] && - dev_priv->wm.spr_latency[1] && - dev_priv->wm.cur_latency[1]) - dev_priv->display.update_wm = ironlake_update_wm; - else { - DRM_DEBUG_KMS("Failed to get proper latency. " - "Disable CxSR\n"); - dev_priv->display.update_wm = NULL; - } + if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] && + dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || + (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] && + dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) { + dev_priv->display.update_wm = ilk_update_wm; + dev_priv->display.update_sprite_wm = ilk_update_sprite_wm; + } else { + DRM_DEBUG_KMS("Failed to read display plane latency. " + "Disable CxSR\n"); + } + + if (IS_GEN5(dev)) dev_priv->display.init_clock_gating = ironlake_init_clock_gating; - } else if (IS_GEN6(dev)) { - if (dev_priv->wm.pri_latency[0] && - dev_priv->wm.spr_latency[0] && - dev_priv->wm.cur_latency[0]) { - dev_priv->display.update_wm = sandybridge_update_wm; - dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm; - } else { - DRM_DEBUG_KMS("Failed to read display plane latency. " - "Disable CxSR\n"); - dev_priv->display.update_wm = NULL; - } + else if (IS_GEN6(dev)) dev_priv->display.init_clock_gating = gen6_init_clock_gating; - } else if (IS_IVYBRIDGE(dev)) { - if (dev_priv->wm.pri_latency[0] && - dev_priv->wm.spr_latency[0] && - dev_priv->wm.cur_latency[0]) { - dev_priv->display.update_wm = ivybridge_update_wm; - dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm; - } else { - DRM_DEBUG_KMS("Failed to read display plane latency. " - "Disable CxSR\n"); - dev_priv->display.update_wm = NULL; - } + else if (IS_IVYBRIDGE(dev)) dev_priv->display.init_clock_gating = ivybridge_init_clock_gating; - } else if (IS_HASWELL(dev)) { - if (dev_priv->wm.pri_latency[0] && - dev_priv->wm.spr_latency[0] && - dev_priv->wm.cur_latency[0]) { - dev_priv->display.update_wm = haswell_update_wm; - dev_priv->display.update_sprite_wm = - haswell_update_sprite_wm; - } else { - DRM_DEBUG_KMS("Failed to read display plane latency. " - "Disable CxSR\n"); - dev_priv->display.update_wm = NULL; - } + else if (IS_HASWELL(dev)) dev_priv->display.init_clock_gating = haswell_init_clock_gating; - } else if (INTEL_INFO(dev)->gen == 8) { + else if (INTEL_INFO(dev)->gen == 8) dev_priv->display.init_clock_gating = gen8_init_clock_gating; - } else - dev_priv->display.update_wm = NULL; } else if (IS_VALLEYVIEW(dev)) { dev_priv->display.update_wm = valleyview_update_wm; dev_priv->display.init_clock_gating = @@ -6026,21 +5692,21 @@ void intel_init_pm(struct drm_device *dev) dev_priv->display.update_wm = i9xx_update_wm; dev_priv->display.get_fifo_size = i9xx_get_fifo_size; dev_priv->display.init_clock_gating = gen3_init_clock_gating; - } else if (IS_I865G(dev)) { - dev_priv->display.update_wm = i830_update_wm; - dev_priv->display.init_clock_gating = i85x_init_clock_gating; - dev_priv->display.get_fifo_size = i830_get_fifo_size; - } else if (IS_I85X(dev)) { - dev_priv->display.update_wm = i9xx_update_wm; - dev_priv->display.get_fifo_size = i85x_get_fifo_size; - dev_priv->display.init_clock_gating = i85x_init_clock_gating; - } else { - dev_priv->display.update_wm = i830_update_wm; - dev_priv->display.init_clock_gating = i830_init_clock_gating; - if (IS_845G(dev)) + } else if (IS_GEN2(dev)) { + if (INTEL_INFO(dev)->num_pipes == 1) { + dev_priv->display.update_wm = i845_update_wm; dev_priv->display.get_fifo_size = i845_get_fifo_size; - else + } else { + dev_priv->display.update_wm = i9xx_update_wm; dev_priv->display.get_fifo_size = i830_get_fifo_size; + } + + if (IS_I85X(dev) || IS_I865G(dev)) + dev_priv->display.init_clock_gating = i85x_init_clock_gating; + else + dev_priv->display.init_clock_gating = i830_init_clock_gating; + } else { + DRM_ERROR("unexpected fall-through in intel_init_pm\n"); } } @@ -6091,59 +5757,48 @@ int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val) return 0; } -int vlv_gpu_freq(int ddr_freq, int val) +int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val) { - int mult, base; + int div; - switch (ddr_freq) { + /* 4 x czclk */ + switch (dev_priv->mem_freq) { case 800: - mult = 20; - base = 120; + div = 10; break; case 1066: - mult = 22; - base = 133; + div = 12; break; case 1333: - mult = 21; - base = 125; + div = 16; break; default: return -1; } - return ((val - 0xbd) * mult) + base; + return DIV_ROUND_CLOSEST(dev_priv->mem_freq * (val + 6 - 0xbd), 4 * div); } -int vlv_freq_opcode(int ddr_freq, int val) +int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val) { - int mult, base; + int mul; - switch (ddr_freq) { + /* 4 x czclk */ + switch (dev_priv->mem_freq) { case 800: - mult = 20; - base = 120; + mul = 10; break; case 1066: - mult = 22; - base = 133; + mul = 12; break; case 1333: - mult = 21; - base = 125; + mul = 16; break; default: return -1; } - val /= mult; - val -= base / mult; - val += 0xbd; - - if (val > 0xea) - val = 0xea; - - return val; + return DIV_ROUND_CLOSEST(4 * mul * val, dev_priv->mem_freq) + 0xbd - 6; } void intel_pm_setup(struct drm_device *dev)