/**
* cik_sdma_ring_emit_vm_flush - cik vm flush using sDMA
*
* @ring: amdgpu_ring pointer
* @vm: amdgpu_vm pointer
*
* Update the page table base and flush the VM TLB
* using sDMA (CIK).
*/
static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring,
unsigned vm_id, uint64_t pd_addr)
{
u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) |
SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */
amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
if (vm_id < 8) {
amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id));
} else {
amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8));
}
amdgpu_ring_write(ring, pd_addr >> 12);
/* flush TLB */
amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
amdgpu_ring_write(ring, 1 << vm_id);
amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);
amdgpu_ring_write(ring, 0);
amdgpu_ring_write(ring, 0); /* reference */
amdgpu_ring_write(ring, 0); /* mask */
amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */
}
/**
* cik_dma_vm_flush - cik vm flush using sDMA
*
* @rdev: radeon_device pointer
*
* Update the page table base and flush the VM TLB
* using sDMA (CIK).
*/
void cik_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
unsigned vm_id, uint64_t pd_addr)
{
u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) |
SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */
radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
if (vm_id < 8) {
radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
} else {
/**
* cik_dma_vm_flush - cik vm flush using sDMA
*
* @rdev: radeon_device pointer
*
* Update the page table base and flush the VM TLB
* using sDMA (CIK).
*/
void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
{
u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) |
SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */
struct radeon_ring *ring = &rdev->ring[ridx];
if (vm == NULL)
return;
radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
if (vm->id < 8) {
radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
} else {
/**
* cik_sdma_ring_emit_pipeline_sync - sync the pipeline
*
* @ring: amdgpu_ring pointer
*
* Make sure all previous operations are completed (CIK).
*/
static void cik_sdma_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
{
uint32_t seq = ring->fence_drv.sync_seq;
uint64_t addr = ring->fence_drv.gpu_addr;
/* wait for idle */
amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0,
SDMA_POLL_REG_MEM_EXTRA_OP(0) |
SDMA_POLL_REG_MEM_EXTRA_FUNC(3) | /* equal */
SDMA_POLL_REG_MEM_EXTRA_M));
amdgpu_ring_write(ring, addr & 0xfffffffc);
amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
amdgpu_ring_write(ring, seq); /* reference */
amdgpu_ring_write(ring, 0xfffffff); /* mask */
amdgpu_ring_write(ring, (0xfff << 16) | 4); /* retry count, poll interval */
}
/**
* cik_sdma_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
*
* @ring: amdgpu ring pointer
*
* Emit an hdp flush packet on the requested DMA ring.
*/
static void cik_sdma_ring_emit_hdp_flush(struct amdgpu_ring *ring)
{
u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
u32 ref_and_mask;
if (ring == &ring->adev->sdma.instance[0].ring)
ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA0_MASK;
else
ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA1_MASK;
amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE << 2);
amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ << 2);
amdgpu_ring_write(ring, ref_and_mask); /* reference */
amdgpu_ring_write(ring, ref_and_mask); /* mask */
amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */
}
/**
* cik_sdma_hdp_flush_ring_emit - emit an hdp flush on the DMA ring
*
* @rdev: radeon_device pointer
* @ridx: radeon ring index
*
* Emit an hdp flush packet on the requested DMA ring.
*/
static void cik_sdma_hdp_flush_ring_emit(struct radeon_device *rdev,
int ridx)
{
struct radeon_ring *ring = &rdev->ring[ridx];
u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
u32 ref_and_mask;
if (ridx == R600_RING_TYPE_DMA_INDEX)
ref_and_mask = SDMA0;
else
ref_and_mask = SDMA1;
radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
radeon_ring_write(ring, ref_and_mask); /* reference */
radeon_ring_write(ring, ref_and_mask); /* mask */
radeon_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */
}
