static void radeon_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *placement) { struct radeon_bo *rbo; static u32 placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; if (!radeon_ttm_bo_is_radeon_bo(bo)) { placement->fpfn = 0; placement->lpfn = 0; placement->placement = &placements; placement->busy_placement = &placements; placement->num_placement = 1; placement->num_busy_placement = 1; return; } rbo = container_of(bo, struct radeon_bo, tbo); switch (bo->mem.mem_type) { case TTM_PL_VRAM: if (rbo->rdev->ring[radeon_copy_ring_index(rbo->rdev)].ready == false) radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); else radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); break; case TTM_PL_TT: default: radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); } *placement = rbo->placement; }
static int radeon_move_blit(struct ttm_buffer_object *bo, bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem, struct ttm_mem_reg *old_mem) { struct radeon_device *rdev; uint64_t old_start, new_start; struct radeon_fence *fence; int r, ridx; rdev = radeon_get_rdev(bo->bdev); ridx = radeon_copy_ring_index(rdev); old_start = old_mem->start << PAGE_SHIFT; new_start = new_mem->start << PAGE_SHIFT; switch (old_mem->mem_type) { case TTM_PL_VRAM: old_start += rdev->mc.vram_start; break; case TTM_PL_TT: old_start += rdev->mc.gtt_start; break; default: DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); return -EINVAL; } switch (new_mem->mem_type) { case TTM_PL_VRAM: new_start += rdev->mc.vram_start; break; case TTM_PL_TT: new_start += rdev->mc.gtt_start; break; default: DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); return -EINVAL; } if (!rdev->ring[ridx].ready) { DRM_ERROR("Trying to move memory with ring turned off.\n"); return -EINVAL; } BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); /* sync other rings */ fence = bo->sync_obj; r = radeon_copy(rdev, old_start, new_start, new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */ &fence); /* FIXME: handle copy error */ r = ttm_bo_move_accel_cleanup(bo, (void *)fence, evict, no_wait_gpu, new_mem); radeon_fence_unref(&fence); return r; }
int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, bool interruptible, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct radeon_device *rdev; struct ttm_mem_reg *old_mem = &bo->mem; int r; rdev = radeon_get_rdev(bo->bdev); if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { radeon_move_null(bo, new_mem); return 0; } if ((old_mem->mem_type == TTM_PL_TT && new_mem->mem_type == TTM_PL_SYSTEM) || (old_mem->mem_type == TTM_PL_SYSTEM && new_mem->mem_type == TTM_PL_TT)) { /* bind is enough */ radeon_move_null(bo, new_mem); return 0; } if (!rdev->ring[radeon_copy_ring_index(rdev)].ready || rdev->asic->copy.copy == NULL) { /* use memcpy */ goto memcpy; } if (old_mem->mem_type == TTM_PL_VRAM && new_mem->mem_type == TTM_PL_SYSTEM) { r = radeon_move_vram_ram(bo, evict, interruptible, no_wait_gpu, new_mem); } else if (old_mem->mem_type == TTM_PL_SYSTEM && new_mem->mem_type == TTM_PL_VRAM) { r = radeon_move_ram_vram(bo, evict, interruptible, no_wait_gpu, new_mem); } else { r = radeon_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem); } if (r) { memcpy: r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); } return r; }
static int radeon_move_blit(struct ttm_buffer_object *bo, bool evict, int no_wait_reserve, bool no_wait_gpu, struct ttm_mem_reg *new_mem, struct ttm_mem_reg *old_mem) { struct radeon_device *rdev; uint64_t old_start, new_start; struct radeon_fence *fence; int r, i; rdev = radeon_get_rdev(bo->bdev); r = radeon_fence_create(rdev, &fence, radeon_copy_ring_index(rdev)); if (unlikely(r)) { return r; } old_start = old_mem->start << PAGE_SHIFT; new_start = new_mem->start << PAGE_SHIFT; switch (old_mem->mem_type) { case TTM_PL_VRAM: old_start += rdev->mc.vram_start; break; case TTM_PL_TT: old_start += rdev->mc.gtt_start; break; default: DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); return -EINVAL; } switch (new_mem->mem_type) { case TTM_PL_VRAM: new_start += rdev->mc.vram_start; break; case TTM_PL_TT: new_start += rdev->mc.gtt_start; break; default: DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); return -EINVAL; } if (!rdev->ring[radeon_copy_ring_index(rdev)].ready) { DRM_ERROR("Trying to move memory with ring turned off.\n"); return -EINVAL; } BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); /* sync other rings */ if (rdev->family >= CHIP_R600) { for (i = 0; i < RADEON_NUM_RINGS; ++i) { /* no need to sync to our own or unused rings */ if (i == radeon_copy_ring_index(rdev) || !rdev->ring[i].ready) continue; if (!fence->semaphore) { r = radeon_semaphore_create(rdev, &fence->semaphore); /* FIXME: handle semaphore error */ if (r) continue; } r = radeon_ring_lock(rdev, &rdev->ring[i], 3); /* FIXME: handle ring lock error */ if (r) continue; radeon_semaphore_emit_signal(rdev, i, fence->semaphore); radeon_ring_unlock_commit(rdev, &rdev->ring[i]); r = radeon_ring_lock(rdev, &rdev->ring[radeon_copy_ring_index(rdev)], 3); /* FIXME: handle ring lock error */ if (r) continue; radeon_semaphore_emit_wait(rdev, radeon_copy_ring_index(rdev), fence->semaphore); radeon_ring_unlock_commit(rdev, &rdev->ring[radeon_copy_ring_index(rdev)]); } } r = radeon_copy(rdev, old_start, new_start, new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */ fence); /* FIXME: handle copy error */ r = ttm_bo_move_accel_cleanup(bo, (void *)fence, NULL, evict, no_wait_reserve, no_wait_gpu, new_mem); radeon_fence_unref(&fence); return r; }