void intel_uncore_early_sanitize(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; if (HAS_FPGA_DBG_UNCLAIMED(dev)) __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); if (IS_HASWELL(dev) && (__raw_i915_read32(dev_priv, HSW_EDRAM_PRESENT) == 1)) { /* The docs do not explain exactly how the calculation can be * made. It is somewhat guessable, but for now, it's always * 128MB. * NB: We can't write IDICR yet because we do not have gt funcs * set up */ dev_priv->ellc_size = 128; DRM_INFO("Found %zuMB of eLLC\n", dev_priv->ellc_size); } /* clear out old GT FIFO errors */ if (IS_GEN6(dev) || IS_GEN7(dev)) __raw_i915_write32(dev_priv, GTFIFODBG, __raw_i915_read32(dev_priv, GTFIFODBG)); intel_uncore_forcewake_reset(dev); }
static void vlv_force_wake_reset(struct drm_i915_private *dev_priv) { __raw_i915_write32(dev_priv, FORCEWAKE_VLV, _MASKED_BIT_DISABLE(0xffff)); __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, _MASKED_BIT_DISABLE(0xffff)); /* something from same cacheline, but !FORCEWAKE_VLV */ __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV); }
static void vlv_force_wake_put(struct drm_i915_private *dev_priv) { __raw_i915_write32(dev_priv, FORCEWAKE_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); /* The below doubles as a POSTING_READ */ gen6_gt_check_fifodbg(dev_priv); }
static void __gen9_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine) { /* Check for Render Engine */ if (FORCEWAKE_RENDER & fw_engine) { if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_RENDER_GEN9) & FORCEWAKE_KERNEL) == 0, FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out: Render forcewake old ack to clear.\n"); __raw_i915_write32(dev_priv, FORCEWAKE_RENDER_GEN9, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_RENDER_GEN9) & FORCEWAKE_KERNEL), FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out: waiting for Render to ack.\n"); } /* Check for Media Engine */ if (FORCEWAKE_MEDIA & fw_engine) { if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_MEDIA_GEN9) & FORCEWAKE_KERNEL) == 0, FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out: Media forcewake old ack to clear.\n"); __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_GEN9, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_MEDIA_GEN9) & FORCEWAKE_KERNEL), FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out: waiting for Media to ack.\n"); } /* Check for Blitter Engine */ if (FORCEWAKE_BLITTER & fw_engine) { if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_BLITTER_GEN9) & FORCEWAKE_KERNEL) == 0, FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out: Blitter forcewake old ack to clear.\n"); __raw_i915_write32(dev_priv, FORCEWAKE_BLITTER_GEN9, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_BLITTER_GEN9) & FORCEWAKE_KERNEL), FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out: waiting for Blitter to ack.\n"); } }
static void __gen9_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv) { __raw_i915_write32(dev_priv, FORCEWAKE_RENDER_GEN9, _MASKED_BIT_DISABLE(0xffff)); __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_GEN9, _MASKED_BIT_DISABLE(0xffff)); __raw_i915_write32(dev_priv, FORCEWAKE_BLITTER_GEN9, _MASKED_BIT_DISABLE(0xffff)); }
void intel_uncore_early_sanitize(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; if (HAS_FPGA_DBG_UNCLAIMED(dev)) __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); }
static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv) { __raw_i915_write32(dev_priv, FORCEWAKE, 0); /* something from same cacheline, but !FORCEWAKE */ __raw_posting_read(dev_priv, ECOBUS); gen6_gt_check_fifodbg(dev_priv); }
static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv, int fw_engine) { u32 forcewake_ack; if (IS_HASWELL(dev_priv->dev) || IS_GEN8(dev_priv->dev)) forcewake_ack = FORCEWAKE_ACK_HSW; else forcewake_ack = FORCEWAKE_MT_ACK; if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL) == 0, FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n"); __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); /* something from same cacheline, but !FORCEWAKE_MT */ __raw_posting_read(dev_priv, ECOBUS); if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL), FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out waiting for forcewake to ack request.\n"); /* WaRsForcewakeWaitTC0:ivb,hsw */ if (INTEL_INFO(dev_priv->dev)->gen < 8) __gen6_gt_wait_for_thread_c0(dev_priv); }
static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv) { __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); /* something from same cacheline, but !FORCEWAKE_MT */ __raw_posting_read(dev_priv, ECOBUS); gen6_gt_check_fifodbg(dev_priv); }
static void hsw_unclaimed_reg_check(struct drm_i915_private *dev_priv, u32 reg) { if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) { DRM_ERROR("Unclaimed write to %x\n", reg); __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); } }
static void ilk_dummy_write(struct drm_i915_private *dev_priv) { /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up * the chip from rc6 before touching it for real. MI_MODE is masked, * hence harmless to write 0 into. */ __raw_i915_write32(dev_priv, MI_MODE, 0); }
void intel_uncore_clear_errors(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; /* XXX needs spinlock around caller's grouping */ if (HAS_FPGA_DBG_UNCLAIMED(dev)) __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); }
static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv) { u32 gtfifodbg; gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG); if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg)) __raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg); }
static void hsw_unclaimed_reg_clear(struct drm_i915_private *dev_priv, u32 reg) { if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) { DRM_ERROR("Unknown unclaimed register before writing to %x\n", reg); __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); } }
static void __gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) { /* Check for Render Engine */ if (FORCEWAKE_RENDER & fw_engine) __raw_i915_write32(dev_priv, FORCEWAKE_RENDER_GEN9, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); /* Check for Media Engine */ if (FORCEWAKE_MEDIA & fw_engine) __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_GEN9, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); /* Check for Blitter Engine */ if (FORCEWAKE_BLITTER & fw_engine) __raw_i915_write32(dev_priv, FORCEWAKE_BLITTER_GEN9, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); }
static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv) { u32 gtfifodbg; gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG); if (WARN(gtfifodbg & GT_FIFO_CPU_ERROR_MASK, "MMIO read or write has been dropped %x\n", gtfifodbg)) __raw_i915_write32(dev_priv, GTFIFODBG, GT_FIFO_CPU_ERROR_MASK); }
static void __vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) { /* Check for Render Engine */ if (FORCEWAKE_RENDER & fw_engine) __raw_i915_write32(dev_priv, FORCEWAKE_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); /* Check for Media Engine */ if (FORCEWAKE_MEDIA & fw_engine) __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); /* The below doubles as a POSTING_READ */ gen6_gt_check_fifodbg(dev_priv); }
static void __vlv_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine) { /* Check for Render Engine */ if (FORCEWAKE_RENDER & fw_engine) { if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_VLV) & FORCEWAKE_KERNEL) == 0, FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out: Render forcewake old ack to clear.\n"); __raw_i915_write32(dev_priv, FORCEWAKE_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_VLV) & FORCEWAKE_KERNEL), FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out: waiting for Render to ack.\n"); } /* Check for Media Engine */ if (FORCEWAKE_MEDIA & fw_engine) { if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_MEDIA_VLV) & FORCEWAKE_KERNEL) == 0, FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out: Media forcewake old ack to clear.\n"); __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_MEDIA_VLV) & FORCEWAKE_KERNEL), FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out: waiting for media to ack.\n"); } /* WaRsForcewakeWaitTC0:vlv */ __gen6_gt_wait_for_thread_c0(dev_priv); }
void intel_uncore_check_errors(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; if (HAS_FPGA_DBG_UNCLAIMED(dev) && (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) { DRM_ERROR("Unclaimed register before interrupt\n"); __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); } }
static void __vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) { /* Check for Render Engine */ if (FORCEWAKE_RENDER & fw_engine) __raw_i915_write32(dev_priv, FORCEWAKE_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); /* Check for Media Engine */ if (FORCEWAKE_MEDIA & fw_engine) __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); /* something from same cacheline, but !FORCEWAKE_VLV */ __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV); if (!IS_CHERRYVIEW(dev_priv->dev)) gen6_gt_check_fifodbg(dev_priv); }
static void hsw_unclaimed_reg_detect(struct drm_i915_private *dev_priv) { if (i915.mmio_debug) return; if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) { DRM_ERROR("Unclaimed register detected. Please use the i915.mmio_debug=1 to debug this problem."); __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); } }
static void vlv_force_wake_get(struct drm_i915_private *dev_priv) { if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_VLV) & FORCEWAKE_KERNEL) == 0, FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n"); __raw_i915_write32(dev_priv, FORCEWAKE_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_VLV) & FORCEWAKE_KERNEL), FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out waiting for GT to ack forcewake request.\n"); if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_MEDIA_VLV) & FORCEWAKE_KERNEL), FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out waiting for media to ack forcewake request.\n"); /* WaRsForcewakeWaitTC0:vlv */ __gen6_gt_wait_for_thread_c0(dev_priv); }
static void hsw_unclaimed_reg_debug(struct drm_i915_private *dev_priv, u32 reg, bool read, bool before) { const char *op = read ? "reading" : "writing to"; const char *when = before ? "before" : "after"; if (!i915.mmio_debug) return; if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) { WARN(1, "Unclaimed register detected %s %s register 0x%x\n", when, op, reg); __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); } }
static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv) { if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0, FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n"); __raw_i915_write32(dev_priv, FORCEWAKE, 1); /* something from same cacheline, but !FORCEWAKE */ __raw_posting_read(dev_priv, ECOBUS); if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1), FORCEWAKE_ACK_TIMEOUT_MS)) DRM_ERROR("Timed out waiting for forcewake to ack request.\n"); /* WaRsForcewakeWaitTC0:snb */ __gen6_gt_wait_for_thread_c0(dev_priv); }
static int gen6_do_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int ret; /* Reset the chip */ /* GEN6_GDRST is not in the gt power well, no need to check * for fifo space for the write or forcewake the chip for * the read */ __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_FULL); /* Spin waiting for the device to ack the reset request */ ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500); intel_uncore_forcewake_reset(dev, true); return ret; }
static int gen6_do_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int ret; unsigned long irqflags; /* Hold uncore.lock across reset to prevent any register access * with forcewake not set correctly */ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); /* Reset the chip */ /* GEN6_GDRST is not in the gt power well, no need to check * for fifo space for the write or forcewake the chip for * the read */ __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_FULL); /* Spin waiting for the device to ack the reset request */ ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500); intel_uncore_forcewake_reset(dev); /* If reset with a user forcewake, try to restore, otherwise turn it off */ if (dev_priv->uncore.forcewake_count) dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL); else dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); /* Restore fifo count */ dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK; spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); return ret; }
static inline void fw_domain_put(const struct intel_uncore_forcewake_domain *d) { __raw_i915_write32(d->i915, d->reg_set, d->val_clear); }
static inline void fw_domain_reset(const struct intel_uncore_forcewake_domain *d) { WARN_ON(!i915_mmio_reg_valid(d->reg_set)); __raw_i915_write32(d->i915, d->reg_set, d->val_reset); }
static void __gen7_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv) { __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff)); /* something from same cacheline, but !FORCEWAKE_MT */ __raw_posting_read(dev_priv, ECOBUS); }