/**
* radeon_ib_get - request an IB (Indirect Buffer)
*
* @rdev: radeon_device pointer
* @ring: ring index the IB is associated with
* @ib: IB object returned
* @size: requested IB size
*
* Request an IB (all asics). IBs are allocated using the
* suballocator.
* Returns 0 on success, error on failure.
*/
int radeon_ib_get(struct radeon_device *rdev, int ring,
struct radeon_ib *ib, struct radeon_vm *vm,
unsigned size)
{
int i, r;
r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &ib->sa_bo, size, 256, true);
if (r) {
dev_err(rdev->dev, "failed to get a new IB (%d)\n", r);
return r;
}
r = radeon_semaphore_create(rdev, &ib->semaphore);
if (r) {
return r;
}
ib->ring = ring;
ib->fence = NULL;
ib->ptr = radeon_sa_bo_cpu_addr(ib->sa_bo);
ib->vm = vm;
if (vm) {
/* ib pool is bound at RADEON_VA_IB_OFFSET in virtual address
* space and soffset is the offset inside the pool bo
*/
ib->gpu_addr = ib->sa_bo->soffset + RADEON_VA_IB_OFFSET;
} else {
ib->gpu_addr = radeon_sa_bo_gpu_addr(ib->sa_bo);
}
ib->is_const_ib = false;
for (i = 0; i < RADEON_NUM_RINGS; ++i)
ib->sync_to[i] = NULL;
return 0;
}
/**
* radeon_ib_get - request an IB (Indirect Buffer)
*
* @rdev: radeon_device pointer
* @ring: ring index the IB is associated with
* @ib: IB object returned
* @size: requested IB size
*
* Request an IB (all asics). IBs are allocated using the
* suballocator.
* Returns 0 on success, error on failure.
*/
int radeon_ib_get(struct radeon_device *rdev, int ring,
struct radeon_ib *ib, unsigned size)
{
int i, r;
r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &ib->sa_bo, size, 256, true);
if (r) {
dev_err(rdev->dev, "failed to get a new IB (%d)\n", r);
return r;
}
r = radeon_semaphore_create(rdev, &ib->semaphore);
if (r) {
return r;
}
ib->ring = ring;
ib->fence = NULL;
ib->ptr = radeon_sa_bo_cpu_addr(ib->sa_bo);
ib->gpu_addr = radeon_sa_bo_gpu_addr(ib->sa_bo);
ib->vm_id = 0;
ib->is_const_ib = false;
for (i = 0; i < RADEON_NUM_RINGS; ++i)
ib->sync_to[i] = NULL;
return 0;
}
/**
* cik_copy_dma - copy pages using the DMA engine
*
* @rdev: radeon_device pointer
* @src_offset: src GPU address
* @dst_offset: dst GPU address
* @num_gpu_pages: number of GPU pages to xfer
* @resv: reservation object to sync to
*
* Copy GPU paging using the DMA engine (CIK).
* Used by the radeon ttm implementation to move pages if
* registered as the asic copy callback.
*/
struct radeon_fence *cik_copy_dma(struct radeon_device *rdev,
uint64_t src_offset, uint64_t dst_offset,
unsigned num_gpu_pages,
struct reservation_object *resv)
{
struct radeon_semaphore *sem = NULL;
struct radeon_fence *fence;
int ring_index = rdev->asic->copy.dma_ring_index;
struct radeon_ring *ring = &rdev->ring[ring_index];
u32 size_in_bytes, cur_size_in_bytes;
int i, num_loops;
int r = 0;
r = radeon_semaphore_create(rdev, &sem);
if (r) {
DRM_ERROR("radeon: moving bo (%d).\n", r);
return ERR_PTR(r);
}
size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14);
if (r) {
DRM_ERROR("radeon: moving bo (%d).\n", r);
radeon_semaphore_free(rdev, &sem, NULL);
return ERR_PTR(r);
}
radeon_semaphore_sync_resv(rdev, sem, resv, false);
radeon_semaphore_sync_rings(rdev, sem, ring->idx);
for (i = 0; i < num_loops; i++) {
cur_size_in_bytes = size_in_bytes;
if (cur_size_in_bytes > 0x1fffff)
cur_size_in_bytes = 0x1fffff;
size_in_bytes -= cur_size_in_bytes;
radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0));
radeon_ring_write(ring, cur_size_in_bytes);
radeon_ring_write(ring, 0); /* src/dst endian swap */
radeon_ring_write(ring, lower_32_bits(src_offset));
radeon_ring_write(ring, upper_32_bits(src_offset));
radeon_ring_write(ring, lower_32_bits(dst_offset));
radeon_ring_write(ring, upper_32_bits(dst_offset));
src_offset += cur_size_in_bytes;
dst_offset += cur_size_in_bytes;
}
r = radeon_fence_emit(rdev, &fence, ring->idx);
if (r) {
radeon_ring_unlock_undo(rdev, ring);
radeon_semaphore_free(rdev, &sem, NULL);
return ERR_PTR(r);
}
radeon_ring_unlock_commit(rdev, ring, false);
radeon_semaphore_free(rdev, &sem, fence);
return fence;
}
/**
* r600_copy_dma - copy pages using the DMA engine
*
* @rdev: radeon_device pointer
* @src_offset: src GPU address
* @dst_offset: dst GPU address
* @num_gpu_pages: number of GPU pages to xfer
* @resv: reservation object to sync to
*
* Copy GPU paging using the DMA engine (r6xx).
* Used by the radeon ttm implementation to move pages if
* registered as the asic copy callback.
*/
struct radeon_fence *r600_copy_dma(struct radeon_device *rdev,
uint64_t src_offset, uint64_t dst_offset,
unsigned num_gpu_pages,
struct reservation_object *resv)
{
struct radeon_semaphore *sem = NULL;
struct radeon_fence *fence;
int ring_index = rdev->asic->copy.dma_ring_index;
struct radeon_ring *ring = &rdev->ring[ring_index];
u32 size_in_dw, cur_size_in_dw;
int i, num_loops;
int r = 0;
r = radeon_semaphore_create(rdev, &sem);
if (r) {
DRM_ERROR("radeon: moving bo (%d).\n", r);
return ERR_PTR(r);
}
size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE);
r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8);
if (r) {
DRM_ERROR("radeon: moving bo (%d).\n", r);
radeon_semaphore_free(rdev, &sem, NULL);
return ERR_PTR(r);
}
radeon_semaphore_sync_resv(rdev, sem, resv, false);
radeon_semaphore_sync_rings(rdev, sem, ring->idx);
for (i = 0; i < num_loops; i++) {
cur_size_in_dw = size_in_dw;
if (cur_size_in_dw > 0xFFFE)
cur_size_in_dw = 0xFFFE;
size_in_dw -= cur_size_in_dw;
radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw));
radeon_ring_write(ring, dst_offset & 0xfffffffc);
radeon_ring_write(ring, src_offset & 0xfffffffc);
radeon_ring_write(ring, (((upper_32_bits(dst_offset) & 0xff) << 16) |
(upper_32_bits(src_offset) & 0xff)));
src_offset += cur_size_in_dw * 4;
dst_offset += cur_size_in_dw * 4;
}
r = radeon_fence_emit(rdev, &fence, ring->idx);
if (r) {
radeon_ring_unlock_undo(rdev, ring);
radeon_semaphore_free(rdev, &sem, NULL);
return ERR_PTR(r);
}
radeon_ring_unlock_commit(rdev, ring, false);
radeon_semaphore_free(rdev, &sem, fence);
return fence;
}
static int radeon_move_blit(struct ttm_buffer_object *bo,
bool evict, int no_wait_reserve, bool no_wait_gpu,
struct ttm_mem_reg *new_mem,
struct ttm_mem_reg *old_mem)
{
struct radeon_device *rdev;
uint64_t old_start, new_start;
struct radeon_fence *fence;
int r, i;
rdev = radeon_get_rdev(bo->bdev);
r = radeon_fence_create(rdev, &fence, radeon_copy_ring_index(rdev));
if (unlikely(r)) {
return r;
}
old_start = old_mem->start << PAGE_SHIFT;
new_start = new_mem->start << PAGE_SHIFT;
switch (old_mem->mem_type) {
case TTM_PL_VRAM:
old_start += rdev->mc.vram_start;
break;
case TTM_PL_TT:
old_start += rdev->mc.gtt_start;
break;
default:
DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
return -EINVAL;
}
switch (new_mem->mem_type) {
case TTM_PL_VRAM:
new_start += rdev->mc.vram_start;
break;
case TTM_PL_TT:
new_start += rdev->mc.gtt_start;
break;
default:
DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
return -EINVAL;
}
if (!rdev->ring[radeon_copy_ring_index(rdev)].ready) {
DRM_ERROR("Trying to move memory with ring turned off.\n");
return -EINVAL;
}
BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0);
/* sync other rings */
if (rdev->family >= CHIP_R600) {
for (i = 0; i < RADEON_NUM_RINGS; ++i) {
/* no need to sync to our own or unused rings */
if (i == radeon_copy_ring_index(rdev) || !rdev->ring[i].ready)
continue;
if (!fence->semaphore) {
r = radeon_semaphore_create(rdev, &fence->semaphore);
/* FIXME: handle semaphore error */
if (r)
continue;
}
r = radeon_ring_lock(rdev, &rdev->ring[i], 3);
/* FIXME: handle ring lock error */
if (r)
continue;
radeon_semaphore_emit_signal(rdev, i, fence->semaphore);
radeon_ring_unlock_commit(rdev, &rdev->ring[i]);
r = radeon_ring_lock(rdev, &rdev->ring[radeon_copy_ring_index(rdev)], 3);
/* FIXME: handle ring lock error */
if (r)
continue;
radeon_semaphore_emit_wait(rdev, radeon_copy_ring_index(rdev), fence->semaphore);
radeon_ring_unlock_commit(rdev, &rdev->ring[radeon_copy_ring_index(rdev)]);
}
}
r = radeon_copy(rdev, old_start, new_start,
new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */
fence);
/* FIXME: handle copy error */
r = ttm_bo_move_accel_cleanup(bo, (void *)fence, NULL,
evict, no_wait_reserve, no_wait_gpu, new_mem);
radeon_fence_unref(&fence);
return r;
}
void radeon_test_ring_sync2(struct radeon_device *rdev,
struct radeon_ring *ringA,
struct radeon_ring *ringB,
struct radeon_ring *ringC)
{
struct radeon_fence *fenceA = NULL, *fenceB = NULL;
struct radeon_semaphore *semaphore = NULL;
int ridxA = radeon_ring_index(rdev, ringA);
int ridxB = radeon_ring_index(rdev, ringB);
int ridxC = radeon_ring_index(rdev, ringC);
bool sigA, sigB;
int i, r;
r = radeon_fence_create(rdev, &fenceA, ridxA);
if (r) {
DRM_ERROR("Failed to create sync fence 1\n");
goto out_cleanup;
}
r = radeon_fence_create(rdev, &fenceB, ridxB);
if (r) {
DRM_ERROR("Failed to create sync fence 2\n");
goto out_cleanup;
}
r = radeon_semaphore_create(rdev, &semaphore);
if (r) {
DRM_ERROR("Failed to create semaphore\n");
goto out_cleanup;
}
r = radeon_ring_lock(rdev, ringA, 64);
if (r) {
DRM_ERROR("Failed to lock ring A %d\n", ridxA);
goto out_cleanup;
}
radeon_semaphore_emit_wait(rdev, ridxA, semaphore);
radeon_fence_emit(rdev, fenceA);
radeon_ring_unlock_commit(rdev, ringA);
r = radeon_ring_lock(rdev, ringB, 64);
if (r) {
DRM_ERROR("Failed to lock ring B %d\n", ridxB);
goto out_cleanup;
}
radeon_semaphore_emit_wait(rdev, ridxB, semaphore);
radeon_fence_emit(rdev, fenceB);
radeon_ring_unlock_commit(rdev, ringB);
mdelay(1000);
if (radeon_fence_signaled(fenceA)) {
DRM_ERROR("Fence A signaled without waiting for semaphore.\n");
goto out_cleanup;
}
if (radeon_fence_signaled(fenceB)) {
DRM_ERROR("Fence A signaled without waiting for semaphore.\n");
goto out_cleanup;
}
r = radeon_ring_lock(rdev, ringC, 64);
if (r) {
DRM_ERROR("Failed to lock ring B %p\n", ringC);
goto out_cleanup;
}
radeon_semaphore_emit_signal(rdev, ridxC, semaphore);
radeon_ring_unlock_commit(rdev, ringC);
for (i = 0; i < 30; ++i) {
mdelay(100);
sigA = radeon_fence_signaled(fenceA);
sigB = radeon_fence_signaled(fenceB);
if (sigA || sigB)
break;
}
if (!sigA && !sigB) {
DRM_ERROR("Neither fence A nor B has been signaled\n");
goto out_cleanup;
} else if (sigA && sigB) {
DRM_ERROR("Both fence A and B has been signaled\n");
goto out_cleanup;
}
DRM_INFO("Fence %c was first signaled\n", sigA ? 'A' : 'B');
r = radeon_ring_lock(rdev, ringC, 64);
if (r) {
DRM_ERROR("Failed to lock ring B %p\n", ringC);
goto out_cleanup;
}
radeon_semaphore_emit_signal(rdev, ridxC, semaphore);
radeon_ring_unlock_commit(rdev, ringC);
mdelay(1000);
r = radeon_fence_wait(fenceA, false);
if (r) {
DRM_ERROR("Failed to wait for sync fence A\n");
goto out_cleanup;
}
r = radeon_fence_wait(fenceB, false);
if (r) {
DRM_ERROR("Failed to wait for sync fence B\n");
goto out_cleanup;
}
out_cleanup:
if (semaphore)
radeon_semaphore_free(rdev, semaphore);
if (fenceA)
radeon_fence_unref(&fenceA);
if (fenceB)
radeon_fence_unref(&fenceB);
if (r)
printk(KERN_WARNING "Error while testing ring sync (%d).\n", r);
}
void radeon_test_ring_sync(struct radeon_device *rdev,
struct radeon_ring *ringA,
struct radeon_ring *ringB)
{
struct radeon_fence *fence1 = NULL, *fence2 = NULL;
struct radeon_semaphore *semaphore = NULL;
int ridxA = radeon_ring_index(rdev, ringA);
int ridxB = radeon_ring_index(rdev, ringB);
int r;
r = radeon_fence_create(rdev, &fence1, ridxA);
if (r) {
DRM_ERROR("Failed to create sync fence 1\n");
goto out_cleanup;
}
r = radeon_fence_create(rdev, &fence2, ridxA);
if (r) {
DRM_ERROR("Failed to create sync fence 2\n");
goto out_cleanup;
}
r = radeon_semaphore_create(rdev, &semaphore);
if (r) {
DRM_ERROR("Failed to create semaphore\n");
goto out_cleanup;
}
r = radeon_ring_lock(rdev, ringA, 64);
if (r) {
DRM_ERROR("Failed to lock ring A %d\n", ridxA);
goto out_cleanup;
}
radeon_semaphore_emit_wait(rdev, ridxA, semaphore);
radeon_fence_emit(rdev, fence1);
radeon_semaphore_emit_wait(rdev, ridxA, semaphore);
radeon_fence_emit(rdev, fence2);
radeon_ring_unlock_commit(rdev, ringA);
mdelay(1000);
if (radeon_fence_signaled(fence1)) {
DRM_ERROR("Fence 1 signaled without waiting for semaphore.\n");
goto out_cleanup;
}
r = radeon_ring_lock(rdev, ringB, 64);
if (r) {
DRM_ERROR("Failed to lock ring B %p\n", ringB);
goto out_cleanup;
}
radeon_semaphore_emit_signal(rdev, ridxB, semaphore);
radeon_ring_unlock_commit(rdev, ringB);
r = radeon_fence_wait(fence1, false);
if (r) {
DRM_ERROR("Failed to wait for sync fence 1\n");
goto out_cleanup;
}
mdelay(1000);
if (radeon_fence_signaled(fence2)) {
DRM_ERROR("Fence 2 signaled without waiting for semaphore.\n");
goto out_cleanup;
}
r = radeon_ring_lock(rdev, ringB, 64);
if (r) {
DRM_ERROR("Failed to lock ring B %p\n", ringB);
goto out_cleanup;
}
radeon_semaphore_emit_signal(rdev, ridxB, semaphore);
radeon_ring_unlock_commit(rdev, ringB);
r = radeon_fence_wait(fence2, false);
if (r) {
DRM_ERROR("Failed to wait for sync fence 1\n");
goto out_cleanup;
}
out_cleanup:
if (semaphore)
radeon_semaphore_free(rdev, semaphore);
if (fence1)
radeon_fence_unref(&fence1);
if (fence2)
radeon_fence_unref(&fence2);
if (r)
printk(KERN_WARNING "Error while testing ring sync (%d).\n", r);
}
void radeon_test_ring_sync(struct radeon_device *rdev,
struct radeon_ring *ringA,
struct radeon_ring *ringB)
{
struct radeon_fence *fence1 = NULL, *fence2 = NULL;
struct radeon_semaphore *semaphore = NULL;
int r;
r = radeon_semaphore_create(rdev, &semaphore);
if (r) {
DRM_ERROR("Failed to create semaphore\n");
goto out_cleanup;
}
r = radeon_ring_lock(rdev, ringA, 64);
if (r) {
DRM_ERROR("Failed to lock ring A %d\n", ringA->idx);
goto out_cleanup;
}
radeon_semaphore_emit_wait(rdev, ringA->idx, semaphore);
r = radeon_fence_emit(rdev, &fence1, ringA->idx);
if (r) {
DRM_ERROR("Failed to emit fence 1\n");
radeon_ring_unlock_undo(rdev, ringA);
goto out_cleanup;
}
radeon_semaphore_emit_wait(rdev, ringA->idx, semaphore);
r = radeon_fence_emit(rdev, &fence2, ringA->idx);
if (r) {
DRM_ERROR("Failed to emit fence 2\n");
radeon_ring_unlock_undo(rdev, ringA);
goto out_cleanup;
}
radeon_ring_unlock_commit(rdev, ringA);
DRM_MDELAY(1000);
if (radeon_fence_signaled(fence1)) {
DRM_ERROR("Fence 1 signaled without waiting for semaphore.\n");
goto out_cleanup;
}
r = radeon_ring_lock(rdev, ringB, 64);
if (r) {
DRM_ERROR("Failed to lock ring B %p\n", ringB);
goto out_cleanup;
}
radeon_semaphore_emit_signal(rdev, ringB->idx, semaphore);
radeon_ring_unlock_commit(rdev, ringB);
r = radeon_fence_wait(fence1, false);
if (r) {
DRM_ERROR("Failed to wait for sync fence 1\n");
goto out_cleanup;
}
DRM_MDELAY(1000);
if (radeon_fence_signaled(fence2)) {
DRM_ERROR("Fence 2 signaled without waiting for semaphore.\n");
goto out_cleanup;
}
r = radeon_ring_lock(rdev, ringB, 64);
if (r) {
DRM_ERROR("Failed to lock ring B %p\n", ringB);
goto out_cleanup;
}
radeon_semaphore_emit_signal(rdev, ringB->idx, semaphore);
radeon_ring_unlock_commit(rdev, ringB);
r = radeon_fence_wait(fence2, false);
if (r) {
DRM_ERROR("Failed to wait for sync fence 1\n");
goto out_cleanup;
}
out_cleanup:
radeon_semaphore_free(rdev, &semaphore, NULL);
if (fence1)
radeon_fence_unref(&fence1);
if (fence2)
radeon_fence_unref(&fence2);
if (r)
DRM_ERROR("Error while testing ring sync (%d).\n", r);
}
static void radeon_test_ring_sync2(struct radeon_device *rdev,
struct radeon_ring *ringA,
struct radeon_ring *ringB,
struct radeon_ring *ringC)
{
struct radeon_fence *fenceA = NULL, *fenceB = NULL;
struct radeon_semaphore *semaphore = NULL;
bool sigA, sigB;
int i, r;
r = radeon_semaphore_create(rdev, &semaphore);
if (r) {
DRM_ERROR("Failed to create semaphore\n");
goto out_cleanup;
}
r = radeon_ring_lock(rdev, ringA, 64);
if (r) {
DRM_ERROR("Failed to lock ring A %d\n", ringA->idx);
goto out_cleanup;
}
radeon_semaphore_emit_wait(rdev, ringA->idx, semaphore);
radeon_ring_unlock_commit(rdev, ringA, false);
r = radeon_test_create_and_emit_fence(rdev, ringA, &fenceA);
if (r)
goto out_cleanup;
r = radeon_ring_lock(rdev, ringB, 64);
if (r) {
DRM_ERROR("Failed to lock ring B %d\n", ringB->idx);
goto out_cleanup;
}
radeon_semaphore_emit_wait(rdev, ringB->idx, semaphore);
radeon_ring_unlock_commit(rdev, ringB, false);
r = radeon_test_create_and_emit_fence(rdev, ringB, &fenceB);
if (r)
goto out_cleanup;
msleep(1000);
if (radeon_fence_signaled(fenceA)) {
DRM_ERROR("Fence A signaled without waiting for semaphore.\n");
goto out_cleanup;
}
if (radeon_fence_signaled(fenceB)) {
DRM_ERROR("Fence B signaled without waiting for semaphore.\n");
goto out_cleanup;
}
r = radeon_ring_lock(rdev, ringC, 64);
if (r) {
DRM_ERROR("Failed to lock ring B %p\n", ringC);
goto out_cleanup;
}
radeon_semaphore_emit_signal(rdev, ringC->idx, semaphore);
radeon_ring_unlock_commit(rdev, ringC, false);
for (i = 0; i < 30; ++i) {
msleep(100);
sigA = radeon_fence_signaled(fenceA);
sigB = radeon_fence_signaled(fenceB);
if (sigA || sigB)
break;
}
if (!sigA && !sigB) {
DRM_ERROR("Neither fence A nor B has been signaled\n");
goto out_cleanup;
} else if (sigA && sigB) {
DRM_ERROR("Both fence A and B has been signaled\n");
goto out_cleanup;
}
DRM_INFO("Fence %c was first signaled\n", sigA ? 'A' : 'B');
r = radeon_ring_lock(rdev, ringC, 64);
if (r) {
DRM_ERROR("Failed to lock ring B %p\n", ringC);
goto out_cleanup;
}
radeon_semaphore_emit_signal(rdev, ringC->idx, semaphore);
radeon_ring_unlock_commit(rdev, ringC, false);
msleep(1000);
r = radeon_fence_wait(fenceA, false);
if (r) {
DRM_ERROR("Failed to wait for sync fence A\n");
goto out_cleanup;
}
r = radeon_fence_wait(fenceB, false);
if (r) {
DRM_ERROR("Failed to wait for sync fence B\n");
goto out_cleanup;
}
out_cleanup:
radeon_semaphore_free(rdev, &semaphore, NULL);
if (fenceA)
radeon_fence_unref(&fenceA);
if (fenceB)
radeon_fence_unref(&fenceB);
if (r)
pr_warn("Error while testing ring sync (%d)\n", r);
}