static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, unsigned vmid, uint64_t pd_addr) { struct amdgpu_device *adev = ring->adev; struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; uint32_t req = gmc_v9_0_get_invalidate_req(vmid); uint64_t flags = AMDGPU_PTE_VALID; unsigned eng = ring->vm_inv_eng; amdgpu_gmc_get_vm_pde(adev, -1, &pd_addr, &flags); pd_addr |= flags; amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid), lower_32_bits(pd_addr)); amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid), upper_32_bits(pd_addr)); amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_req + eng, req); /* wait for the invalidate to complete */ amdgpu_ring_emit_reg_wait(ring, hub->vm_inv_eng0_ack + eng, 1 << vmid, 1 << vmid); return pd_addr; }
static void soc15_invalidate_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring) { if (!ring || !ring->funcs->emit_wreg) WREG32_SOC15_NO_KIQ(NBIO, 0, mmHDP_READ_CACHE_INVALIDATE, 1); else amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1); }
static void nbio_v7_4_hdp_flush(struct amdgpu_device *adev, struct amdgpu_ring *ring) { if (!ring || !ring->funcs->emit_wreg) WREG32_SOC15_NO_KIQ(NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0); else amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL), 0); }
static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, unsigned pasid) { struct amdgpu_device *adev = ring->adev; uint32_t reg; if (ring->funcs->vmhub == AMDGPU_GFXHUB) reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; else reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; amdgpu_ring_emit_wreg(ring, reg, pasid); }
void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) { signed long r, cnt = 0; unsigned long flags; uint32_t seq; struct amdgpu_kiq *kiq = &adev->gfx.kiq; struct amdgpu_ring *ring = &kiq->ring; BUG_ON(!ring->funcs->emit_wreg); spin_lock_irqsave(&kiq->ring_lock, flags); amdgpu_ring_alloc(ring, 32); amdgpu_ring_emit_wreg(ring, reg, v); amdgpu_fence_emit_polling(ring, &seq); amdgpu_ring_commit(ring); spin_unlock_irqrestore(&kiq->ring_lock, flags); r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); /* don't wait anymore for gpu reset case because this way may * block gpu_recover() routine forever, e.g. this virt_kiq_rreg * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will * never return if we keep waiting in virt_kiq_rreg, which cause * gpu_recover() hang there. * * also don't wait anymore for IRQ context * */ if (r < 1 && (adev->in_gpu_reset || in_interrupt())) goto failed_kiq_write; might_sleep(); while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); } if (cnt > MAX_KIQ_REG_TRY) goto failed_kiq_write; return; failed_kiq_write: pr_err("failed to write reg:%x\n", reg); }
void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) { signed long r; struct dma_fence *f; struct amdgpu_kiq *kiq = &adev->gfx.kiq; struct amdgpu_ring *ring = &kiq->ring; BUG_ON(!ring->funcs->emit_wreg); mutex_lock(&adev->virt.lock_kiq); amdgpu_ring_alloc(ring, 32); amdgpu_ring_emit_wreg(ring, reg, v); amdgpu_fence_emit(ring, &f); amdgpu_ring_commit(ring); mutex_unlock(&adev->virt.lock_kiq); r = dma_fence_wait(f, false); if (r) DRM_ERROR("wait for kiq fence error: %ld.\n", r); dma_fence_put(f); }
void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) { signed long r; unsigned long flags; uint32_t seq; struct amdgpu_kiq *kiq = &adev->gfx.kiq; struct amdgpu_ring *ring = &kiq->ring; BUG_ON(!ring->funcs->emit_wreg); spin_lock_irqsave(&kiq->ring_lock, flags); amdgpu_ring_alloc(ring, 32); amdgpu_ring_emit_wreg(ring, reg, v); amdgpu_fence_emit_polling(ring, &seq); amdgpu_ring_commit(ring); spin_unlock_irqrestore(&kiq->ring_lock, flags); r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); if (r < 1) DRM_ERROR("wait for kiq fence error: %ld\n", r); }