/** * cik_sdma_ring_emit_vm_flush - cik vm flush using sDMA * * @ring: amdgpu_ring pointer * @vm: amdgpu_vm pointer * * Update the page table base and flush the VM TLB * using sDMA (CIK). */ static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring, unsigned vm_id, uint64_t pd_addr) { u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) | SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */ amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); if (vm_id < 8) { amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id)); } else { amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8)); } amdgpu_ring_write(ring, pd_addr >> 12); /* flush TLB */ amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST); amdgpu_ring_write(ring, 1 << vm_id); amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits)); amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2); amdgpu_ring_write(ring, 0); amdgpu_ring_write(ring, 0); /* reference */ amdgpu_ring_write(ring, 0); /* mask */ amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */ }
/** * cik_dma_vm_flush - cik vm flush using sDMA * * @rdev: radeon_device pointer * * Update the page table base and flush the VM TLB * using sDMA (CIK). */ void cik_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, unsigned vm_id, uint64_t pd_addr) { u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) | SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */ radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); if (vm_id < 8) { radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); } else {
/** * cik_dma_vm_flush - cik vm flush using sDMA * * @rdev: radeon_device pointer * * Update the page table base and flush the VM TLB * using sDMA (CIK). */ void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) { u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) | SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */ struct radeon_ring *ring = &rdev->ring[ridx]; if (vm == NULL) return; radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); if (vm->id < 8) { radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); } else {
/** * cik_sdma_ring_emit_pipeline_sync - sync the pipeline * * @ring: amdgpu_ring pointer * * Make sure all previous operations are completed (CIK). */ static void cik_sdma_ring_emit_pipeline_sync(struct amdgpu_ring *ring) { uint32_t seq = ring->fence_drv.sync_seq; uint64_t addr = ring->fence_drv.gpu_addr; /* wait for idle */ amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, SDMA_POLL_REG_MEM_EXTRA_OP(0) | SDMA_POLL_REG_MEM_EXTRA_FUNC(3) | /* equal */ SDMA_POLL_REG_MEM_EXTRA_M)); amdgpu_ring_write(ring, addr & 0xfffffffc); amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); amdgpu_ring_write(ring, seq); /* reference */ amdgpu_ring_write(ring, 0xfffffff); /* mask */ amdgpu_ring_write(ring, (0xfff << 16) | 4); /* retry count, poll interval */ }
/** * cik_sdma_ring_emit_hdp_flush - emit an hdp flush on the DMA ring * * @ring: amdgpu ring pointer * * Emit an hdp flush packet on the requested DMA ring. */ static void cik_sdma_ring_emit_hdp_flush(struct amdgpu_ring *ring) { u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) | SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */ u32 ref_and_mask; if (ring == &ring->adev->sdma.instance[0].ring) ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA0_MASK; else ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA1_MASK; amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits)); amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE << 2); amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ << 2); amdgpu_ring_write(ring, ref_and_mask); /* reference */ amdgpu_ring_write(ring, ref_and_mask); /* mask */ amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */ }
/** * cik_sdma_hdp_flush_ring_emit - emit an hdp flush on the DMA ring * * @rdev: radeon_device pointer * @ridx: radeon ring index * * Emit an hdp flush packet on the requested DMA ring. */ static void cik_sdma_hdp_flush_ring_emit(struct radeon_device *rdev, int ridx) { struct radeon_ring *ring = &rdev->ring[ridx]; u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) | SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */ u32 ref_and_mask; if (ridx == R600_RING_TYPE_DMA_INDEX) ref_and_mask = SDMA0; else ref_and_mask = SDMA1; radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits)); radeon_ring_write(ring, GPU_HDP_FLUSH_DONE); radeon_ring_write(ring, GPU_HDP_FLUSH_REQ); radeon_ring_write(ring, ref_and_mask); /* reference */ radeon_ring_write(ring, ref_and_mask); /* mask */ radeon_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */ }