static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt = (void *)ttm;
unsigned i;
bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
if (slave)
return;
rdev = radeon_get_rdev(ttm->bdev);
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
ttm_agp_tt_unpopulate(ttm);
return;
}
#endif
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
ttm_dma_unpopulate(>t->ttm, rdev->dev);
return;
}
#endif
bus_dmamap_unload(rdev->dmat, gtt->map);
for (i = 0; i < ttm->num_pages; i++)
gtt->ttm.dma_address[i] = 0;
ttm_pool_unpopulate(ttm);
}
struct ttm_tt *radeon_ttm_tt_create(struct ttm_bo_device *bdev,
unsigned long size, uint32_t page_flags,
struct page *dummy_read_page)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt;
rdev = radeon_get_rdev(bdev);
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_tt_create(bdev, rdev->ddev->agp->bridge,
size, page_flags, dummy_read_page);
}
#endif
gtt = kzalloc(sizeof(struct radeon_ttm_tt), GFP_KERNEL);
if (gtt == NULL) {
return NULL;
}
gtt->ttm.ttm.func = &radeon_backend_func;
gtt->rdev = rdev;
if (ttm_dma_tt_init(>t->ttm, bdev, size, page_flags, dummy_read_page)) {
kfree(gtt);
return NULL;
}
return >t->ttm.ttm;
}
static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt = (void *)ttm;
unsigned i;
rdev = radeon_get_rdev(ttm->bdev);
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
ttm_agp_tt_unpopulate(ttm);
return;
}
#endif
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
ttm_dma_unpopulate(>t->ttm, rdev->dev);
return;
}
#endif
for (i = 0; i < ttm->num_pages; i++) {
if (gtt->ttm.dma_address[i]) {
pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i],
PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
}
}
ttm_pool_unpopulate(ttm);
}
static struct ttm_tt *radeon_ttm_tt_create(struct ttm_bo_device *bdev,
unsigned long size, uint32_t page_flags,
vm_page_t dummy_read_page)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt;
rdev = radeon_get_rdev(bdev);
#if __OS_HAS_AGP
#ifdef DUMBBELL_WIP
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_tt_create(bdev, rdev->ddev->agp->agpdev,
size, page_flags, dummy_read_page);
}
#endif /* DUMBBELL_WIP */
#endif
gtt = kmalloc(sizeof(struct radeon_ttm_tt), DRM_MEM_DRIVER,
M_WAITOK | M_ZERO);
if (gtt == NULL) {
return NULL;
}
gtt->ttm.ttm.func = &radeon_backend_func;
gtt->rdev = rdev;
if (ttm_dma_tt_init(>t->ttm, bdev, size, page_flags, dummy_read_page)) {
drm_free(gtt, DRM_MEM_DRIVER);
return NULL;
}
return >t->ttm.ttm;
}
static int radeon_move_blit(struct ttm_buffer_object *bo,
bool evict, int no_wait_reserve, bool no_wait_gpu,
struct ttm_mem_reg *new_mem,
struct ttm_mem_reg *old_mem)
{
struct radeon_device *rdev;
uint64_t old_start, new_start;
struct radeon_fence *fence;
int r;
rdev = radeon_get_rdev(bo->bdev);
r = radeon_fence_create(rdev, &fence);
if (unlikely(r)) {
return r;
}
old_start = old_mem->start << PAGE_SHIFT;
new_start = new_mem->start << PAGE_SHIFT;
switch (old_mem->mem_type) {
case TTM_PL_VRAM:
old_start += rdev->mc.vram_start;
break;
case TTM_PL_TT:
old_start += rdev->mc.gtt_start;
break;
default:
DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
return -EINVAL;
}
switch (new_mem->mem_type) {
case TTM_PL_VRAM:
new_start += rdev->mc.vram_start;
break;
case TTM_PL_TT:
new_start += rdev->mc.gtt_start;
break;
default:
DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
return -EINVAL;
}
if (!rdev->cp.ready) {
DRM_ERROR("Trying to move memory with CP turned off.\n");
return -EINVAL;
}
BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0);
r = radeon_copy(rdev, old_start, new_start,
new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */
fence);
/* FIXME: handle copy error */
r = ttm_bo_move_accel_cleanup(bo, (void *)fence, NULL,
evict, no_wait_reserve, no_wait_gpu, new_mem);
radeon_fence_unref(&fence);
return r;
}
static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
struct ttm_mem_type_manager *man)
{
struct radeon_device *rdev;
ENTER();
rdev = radeon_get_rdev(bdev);
switch (type) {
case TTM_PL_SYSTEM:
/* System memory */
man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
man->available_caching = TTM_PL_MASK_CACHING;
man->default_caching = TTM_PL_FLAG_CACHED;
break;
case TTM_PL_TT:
man->func = &ttm_bo_manager_func;
man->gpu_offset = rdev->mc.gtt_start;
man->available_caching = TTM_PL_MASK_CACHING;
man->default_caching = TTM_PL_FLAG_CACHED;
man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
if (!(drm_core_has_AGP(rdev->ddev) && rdev->ddev->agp)) {
DRM_ERROR("AGP is not enabled for memory type %u\n",
(unsigned)type);
return -EINVAL;
}
if (!rdev->ddev->agp->cant_use_aperture)
man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
man->available_caching = TTM_PL_FLAG_UNCACHED |
TTM_PL_FLAG_WC;
man->default_caching = TTM_PL_FLAG_WC;
}
#endif
break;
case TTM_PL_VRAM:
/* "On-card" video ram */
man->func = &ttm_bo_manager_func;
man->gpu_offset = rdev->mc.vram_start;
man->flags = TTM_MEMTYPE_FLAG_FIXED |
TTM_MEMTYPE_FLAG_MAPPABLE;
man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
man->default_caching = TTM_PL_FLAG_WC;
break;
default:
DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
return -EINVAL;
}
LEAVE();
return 0;
}
static int radeon_move_blit(struct ttm_buffer_object *bo,
bool evict, bool no_wait_gpu,
struct ttm_mem_reg *new_mem,
struct ttm_mem_reg *old_mem)
{
struct radeon_device *rdev;
uint64_t old_start, new_start;
struct radeon_fence *fence;
int r, ridx;
rdev = radeon_get_rdev(bo->bdev);
ridx = radeon_copy_ring_index(rdev);
old_start = old_mem->start << PAGE_SHIFT;
new_start = new_mem->start << PAGE_SHIFT;
switch (old_mem->mem_type) {
case TTM_PL_VRAM:
old_start += rdev->mc.vram_start;
break;
case TTM_PL_TT:
old_start += rdev->mc.gtt_start;
break;
default:
DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
return -EINVAL;
}
switch (new_mem->mem_type) {
case TTM_PL_VRAM:
new_start += rdev->mc.vram_start;
break;
case TTM_PL_TT:
new_start += rdev->mc.gtt_start;
break;
default:
DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
return -EINVAL;
}
if (!rdev->ring[ridx].ready) {
DRM_ERROR("Trying to move memory with ring turned off.\n");
return -EINVAL;
}
CTASSERT((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) == 0);
/* sync other rings */
fence = bo->sync_obj;
r = radeon_copy(rdev, old_start, new_start,
new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */
&fence);
/* FIXME: handle copy error */
r = ttm_bo_move_accel_cleanup(bo, (void *)fence,
evict, no_wait_gpu, new_mem);
radeon_fence_unref(&fence);
return r;
}
static int radeon_ttm_tt_populate(struct ttm_tt *ttm)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt = (void *)ttm;
unsigned i;
int r;
bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
if (ttm->state != tt_unpopulated)
return 0;
if (slave && ttm->sg) {
drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
gtt->ttm.dma_address, ttm->num_pages);
ttm->state = tt_unbound;
return 0;
}
rdev = radeon_get_rdev(ttm->bdev);
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_tt_populate(ttm);
}
#endif
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0))
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
return ttm_dma_populate(>t->ttm, rdev->dev);
}
#endif
#endif
r = ttm_pool_populate(ttm);
if (r) {
return r;
}
for (i = 0; i < ttm->num_pages; i++) {
gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i],
0, PAGE_SIZE,
PCI_DMA_BIDIRECTIONAL);
if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) {
while (--i) {
pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i],
PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
gtt->ttm.dma_address[i] = 0;
}
ttm_pool_unpopulate(ttm);
return -EFAULT;
}
}
return 0;
}
static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
bool evict, bool interruptible,
bool no_wait_reserve, bool no_wait_gpu,
struct ttm_mem_reg *new_mem)
{
struct radeon_device *rdev;
struct ttm_mem_reg *old_mem = &bo->mem;
struct ttm_mem_reg tmp_mem;
u32 placements;
struct ttm_placement placement;
int r;
rdev = radeon_get_rdev(bo->bdev);
tmp_mem = *new_mem;
tmp_mem.mm_node = NULL;
placement.fpfn = 0;
placement.lpfn = 0;
placement.num_placement = 1;
placement.placement = &placements;
placement.num_busy_placement = 1;
placement.busy_placement = &placements;
placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
interruptible, no_wait_reserve, no_wait_gpu);
if (unlikely(r)) {
return r;
}
r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
if (unlikely(r)) {
goto out_cleanup;
}
r = ttm_tt_bind(bo->ttm, &tmp_mem);
if (unlikely(r)) {
goto out_cleanup;
}
r = radeon_move_blit(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem, old_mem);
if (unlikely(r)) {
goto out_cleanup;
}
r = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem);
out_cleanup:
if (tmp_mem.mm_node) {
struct ttm_bo_global *glob = rdev->mman.bdev.glob;
spin_lock(&glob->lru_lock);
drm_mm_put_block(tmp_mem.mm_node);
spin_unlock(&glob->lru_lock);
return r;
}
return r;
}
static int radeon_move_blit(struct ttm_buffer_object *bo,
bool evict, int no_wait,
struct ttm_mem_reg *new_mem,
struct ttm_mem_reg *old_mem)
{
struct radeon_device *rdev;
uint64_t old_start, new_start;
struct radeon_fence *fence;
int r;
rdev = radeon_get_rdev(bo->bdev);
r = radeon_fence_create(rdev, &fence);
if (unlikely(r)) {
return r;
}
old_start = old_mem->mm_node->start << PAGE_SHIFT;
new_start = new_mem->mm_node->start << PAGE_SHIFT;
switch (old_mem->mem_type) {
case TTM_PL_VRAM:
old_start += rdev->mc.vram_location;
break;
case TTM_PL_TT:
old_start += rdev->mc.gtt_location;
break;
default:
DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
return -EINVAL;
}
switch (new_mem->mem_type) {
case TTM_PL_VRAM:
new_start += rdev->mc.vram_location;
break;
case TTM_PL_TT:
new_start += rdev->mc.gtt_location;
break;
default:
DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
return -EINVAL;
}
if (!rdev->cp.ready) {
DRM_ERROR("Trying to move memory with CP turned off.\n");
return -EINVAL;
}
r = radeon_copy(rdev, old_start, new_start, new_mem->num_pages, fence);
r = ttm_bo_move_accel_cleanup(bo, (void *)fence, NULL,
evict, no_wait, new_mem);
radeon_fence_unref(&fence);
return r;
}
static struct ttm_backend*
radeon_create_ttm_backend_entry(struct ttm_bo_device *bdev)
{
struct radeon_device *rdev;
rdev = radeon_get_rdev(bdev);
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_backend_init(bdev, rdev->ddev->agp->bridge);
} else
#endif
{
return radeon_ttm_backend_create(rdev);
}
}
static int radeon_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
struct ttm_buffer_object *bo;
struct radeon_device *rdev;
int r;
bo = (struct ttm_buffer_object *)vma->vm_private_data;
if (bo == NULL) {
return VM_FAULT_NOPAGE;
}
rdev = radeon_get_rdev(bo->bdev);
mutex_lock(&rdev->vram_mutex);
r = ttm_vm_ops->fault(vma, vmf);
mutex_unlock(&rdev->vram_mutex);
return r;
}
static int radeon_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
struct ttm_buffer_object *bo;
struct radeon_device *rdev;
int r;
bo = (struct ttm_buffer_object *)vma->vm_private_data;
if (bo == NULL) {
return VM_FAULT_NOPAGE;
}
rdev = radeon_get_rdev(bo->bdev);
lockmgr(&rdev->pm.mclk_lock, LK_SHARED);
r = ttm_vm_ops->fault(vma, vmf);
lockmgr(&rdev->pm.mclk_lock, LK_RELEASE);
return r;
}
int
radeon_bo_move(struct ttm_buffer_object *bo,
bool evict, bool interruptible,
bool no_wait_gpu,
struct ttm_mem_reg *new_mem)
{
struct radeon_device *rdev;
struct ttm_mem_reg *old_mem = &bo->mem;
int r;
rdev = radeon_get_rdev(bo->bdev);
if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
radeon_move_null(bo, new_mem);
return 0;
}
if ((old_mem->mem_type == TTM_PL_TT &&
new_mem->mem_type == TTM_PL_SYSTEM) ||
(old_mem->mem_type == TTM_PL_SYSTEM &&
new_mem->mem_type == TTM_PL_TT)) {
/* bind is enough */
radeon_move_null(bo, new_mem);
return 0;
}
if (!rdev->ring[radeon_copy_ring_index(rdev)].ready ||
rdev->asic->copy.copy == NULL) {
/* use memcpy */
goto memcpy;
}
if (old_mem->mem_type == TTM_PL_VRAM &&
new_mem->mem_type == TTM_PL_SYSTEM) {
r = radeon_move_vram_ram(bo, evict, interruptible,
no_wait_gpu, new_mem);
} else if (old_mem->mem_type == TTM_PL_SYSTEM &&
new_mem->mem_type == TTM_PL_VRAM) {
r = radeon_move_ram_vram(bo, evict, interruptible,
no_wait_gpu, new_mem);
} else {
r = radeon_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem);
}
if (r) {
memcpy:
r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
}
return r;
}
static int
radeon_ttm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, vm_page_t *pps,
int npages, int centeridx, vm_fault_t fault_type,
vm_prot_t access_type, int flags)
{
struct ttm_buffer_object *bo;
struct radeon_device *rdev;
int r;
bo = (struct ttm_buffer_object *)ufi->entry->object.uvm_obj;
rdev = radeon_get_rdev(bo->bdev);
down_read(&rdev->pm.mclk_lock);
r = ttm_vm_ops->pgo_fault(ufi, vaddr, pps, npages, centeridx,
fault_type, access_type, flags);
up_read(&rdev->pm.mclk_lock);
return r;
}
static int radeon_ttm_tt_populate(struct ttm_tt *ttm)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt = (void *)ttm;
unsigned i;
int r;
if (ttm->state != tt_unpopulated)
return 0;
rdev = radeon_get_rdev(ttm->bdev);
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_tt_populate(ttm);
}
#endif
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
return ttm_dma_populate(>t->ttm, rdev->dev);
}
#endif
r = ttm_pool_populate(ttm);
if (r) {
return r;
}
for (i = 0; i < ttm->num_pages; i++) {
gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i],
0, PAGE_SIZE,
PCI_DMA_BIDIRECTIONAL);
if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) {
while (--i) {
pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i],
PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
gtt->ttm.dma_address[i] = 0;
}
ttm_pool_unpopulate(ttm);
return -EFAULT;
}
}
return 0;
}
static struct ttm_tt *radeon_ttm_tt_create(struct ttm_bo_device *bdev,
unsigned long size, uint32_t page_flags,
struct vm_page *dummy_read_page)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt;
rdev = radeon_get_rdev(bdev);
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_tt_create(bdev, rdev->ddev->agp,
size, page_flags, dummy_read_page);
}
#endif
gtt = kzalloc(sizeof(struct radeon_ttm_tt), GFP_KERNEL);
if (gtt == NULL) {
return NULL;
}
gtt->ttm.ttm.func = &radeon_backend_func;
gtt->rdev = rdev;
if (ttm_dma_tt_init(>t->ttm, bdev, size, page_flags, dummy_read_page)) {
kfree(gtt);
return NULL;
}
gtt->segs = mallocarray(gtt->ttm.ttm.num_pages,
sizeof(bus_dma_segment_t), M_DRM, M_WAITOK | M_ZERO);
if (gtt->segs == NULL) {
ttm_dma_tt_fini(>t->ttm);
free(gtt, M_DRM, 0);
return NULL;
}
if (bus_dmamap_create(rdev->dmat, size, gtt->ttm.ttm.num_pages, size,
0, BUS_DMA_WAITOK, >t->map)) {
free(gtt->segs, M_DRM, 0);
ttm_dma_tt_fini(>t->ttm);
free(gtt, M_DRM, 0);
return NULL;
}
return >t->ttm.ttm;
}
static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
bool evict, bool interruptible,
bool no_wait_gpu,
struct ttm_mem_reg *new_mem)
{
struct radeon_device *rdev;
struct ttm_mem_reg *old_mem = &bo->mem;
struct ttm_mem_reg tmp_mem;
struct ttm_placement placement;
u32 placements;
int r;
rdev = radeon_get_rdev(bo->bdev);
tmp_mem = *new_mem;
tmp_mem.mm_node = NULL;
placement.fpfn = 0;
placement.lpfn = 0;
placement.num_placement = 1;
placement.placement = &placements;
placement.num_busy_placement = 1;
placement.busy_placement = &placements;
placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
interruptible, no_wait_gpu);
if (unlikely(r)) {
return r;
}
r = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem);
if (unlikely(r)) {
goto out_cleanup;
}
r = radeon_move_blit(bo, true, no_wait_gpu, new_mem, old_mem);
if (unlikely(r)) {
goto out_cleanup;
}
out_cleanup:
ttm_bo_mem_put(bo, &tmp_mem);
return r;
}
static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
struct radeon_device *rdev = radeon_get_rdev(bdev);
mem->bus.addr = NULL;
mem->bus.offset = 0;
mem->bus.size = mem->num_pages << PAGE_SHIFT;
mem->bus.base = 0;
mem->bus.is_iomem = false;
if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
return -EINVAL;
switch (mem->mem_type) {
case TTM_PL_SYSTEM:
/* system memory */
return 0;
case TTM_PL_TT:
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
/* RADEON_IS_AGP is set only if AGP is active */
mem->bus.offset = mem->start << PAGE_SHIFT;
mem->bus.base = rdev->mc.agp_base;
mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture;
}
#endif
break;
case TTM_PL_VRAM:
mem->bus.offset = mem->start << PAGE_SHIFT;
/* check if it's visible */
if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size)
return -EINVAL;
mem->bus.base = rdev->mc.aper_base;
mem->bus.is_iomem = true;
break;
default:
return -EINVAL;
}
return 0;
}
static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
bool evict, bool interruptible, bool no_wait,
struct ttm_mem_reg *new_mem)
{
struct radeon_device *rdev;
struct ttm_mem_reg *old_mem = &bo->mem;
struct ttm_mem_reg tmp_mem;
uint32_t proposed_flags;
int r;
rdev = radeon_get_rdev(bo->bdev);
tmp_mem = *new_mem;
tmp_mem.mm_node = NULL;
proposed_flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
r = ttm_bo_mem_space(bo, proposed_flags, &tmp_mem,
interruptible, no_wait);
if (unlikely(r)) {
return r;
}
r = ttm_bo_move_ttm(bo, true, no_wait, &tmp_mem);
if (unlikely(r)) {
goto out_cleanup;
}
r = radeon_move_blit(bo, true, no_wait, new_mem, old_mem);
if (unlikely(r)) {
goto out_cleanup;
}
out_cleanup:
if (tmp_mem.mm_node) {
struct ttm_bo_global *glob = rdev->mman.bdev.glob;
spin_lock(&glob->lru_lock);
drm_mm_put_block(tmp_mem.mm_node);
spin_unlock(&glob->lru_lock);
return r;
}
return r;
}
static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
struct radeon_device *rdev = radeon_get_rdev(bdev);
mem->bus.addr = NULL;
mem->bus.offset = 0;
mem->bus.size = mem->num_pages << PAGE_SHIFT;
mem->bus.base = 0;
mem->bus.is_iomem = false;
if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
return -EINVAL;
switch (mem->mem_type) {
case TTM_PL_SYSTEM:
/* */
return 0;
case TTM_PL_TT:
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
/* */
mem->bus.offset = mem->start << PAGE_SHIFT;
mem->bus.base = rdev->mc.agp_base;
mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture;
}
#endif
break;
case TTM_PL_VRAM:
mem->bus.offset = mem->start << PAGE_SHIFT;
/* */
if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size)
return -EINVAL;
mem->bus.base = rdev->mc.aper_base;
mem->bus.is_iomem = true;
#ifdef __alpha__
/*
*/
if (mem->placement & TTM_PL_FLAG_WC)
mem->bus.addr =
ioremap_wc(mem->bus.base + mem->bus.offset,
mem->bus.size);
else
mem->bus.addr =
ioremap_nocache(mem->bus.base + mem->bus.offset,
mem->bus.size);
/*
*/
mem->bus.base = (mem->bus.base & 0x0ffffffffUL) +
rdev->ddev->hose->dense_mem_base;
#endif
break;
default:
return -EINVAL;
}
return 0;
}
static int radeon_ttm_tt_populate(struct ttm_tt *ttm)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt = (void *)ttm;
unsigned i;
int r;
#ifdef DUMBBELL_WIP
bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
#endif /* DUMBBELL_WIP */
if (ttm->state != tt_unpopulated)
return 0;
#ifdef DUMBBELL_WIP
/*
* Maybe unneeded on FreeBSD.
* -- dumbbell@
*/
if (slave && ttm->sg) {
drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
gtt->ttm.dma_address, ttm->num_pages);
ttm->state = tt_unbound;
return 0;
}
#endif /* DUMBBELL_WIP */
rdev = radeon_get_rdev(ttm->bdev);
#if __OS_HAS_AGP
#ifdef DUMBBELL_WIP
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_tt_populate(ttm);
}
#endif /* DUMBBELL_WIP */
#endif
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
return ttm_dma_populate(>t->ttm, rdev->dev);
}
#endif
r = ttm_pool_populate(ttm);
if (r) {
return r;
}
for (i = 0; i < ttm->num_pages; i++) {
gtt->ttm.dma_address[i] = VM_PAGE_TO_PHYS(ttm->pages[i]);
#ifdef DUMBBELL_WIP
gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i],
0, PAGE_SIZE,
PCI_DMA_BIDIRECTIONAL);
if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) {
while (--i) {
pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i],
PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
gtt->ttm.dma_address[i] = 0;
}
ttm_pool_unpopulate(ttm);
return -EFAULT;
}
#endif /* DUMBBELL_WIP */
}
return 0;
}
static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
struct radeon_device *rdev = radeon_get_rdev(bdev);
mem->bus.addr = NULL;
mem->bus.offset = 0;
mem->bus.size = mem->num_pages << PAGE_SHIFT;
mem->bus.base = 0;
mem->bus.is_iomem = false;
if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
return -EINVAL;
switch (mem->mem_type) {
case TTM_PL_SYSTEM:
/* system memory */
return 0;
case TTM_PL_TT:
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
/* RADEON_IS_AGP is set only if AGP is active */
mem->bus.offset = mem->start << PAGE_SHIFT;
mem->bus.base = rdev->mc.agp_base;
mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture;
}
#endif
break;
case TTM_PL_VRAM:
mem->bus.offset = mem->start << PAGE_SHIFT;
/* check if it's visible */
if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size)
return -EINVAL;
mem->bus.base = rdev->mc.aper_base;
mem->bus.is_iomem = true;
#ifdef __alpha__
/*
* Alpha: use bus.addr to hold the ioremap() return,
* so we can modify bus.base below.
*/
if (mem->placement & TTM_PL_FLAG_WC)
mem->bus.addr =
ioremap_wc(mem->bus.base + mem->bus.offset,
mem->bus.size);
else
mem->bus.addr =
ioremap_nocache(mem->bus.base + mem->bus.offset,
mem->bus.size);
/*
* Alpha: Use just the bus offset plus
* the hose/domain memory base for bus.base.
* It then can be used to build PTEs for VRAM
* access, as done in ttm_bo_vm_fault().
*/
mem->bus.base = (mem->bus.base & 0x0ffffffffUL) +
rdev->ddev->hose->dense_mem_base;
#endif
break;
default:
return -EINVAL;
}
return 0;
}
static int radeon_move_blit(struct ttm_buffer_object *bo,
bool evict, int no_wait_reserve, bool no_wait_gpu,
struct ttm_mem_reg *new_mem,
struct ttm_mem_reg *old_mem)
{
struct radeon_device *rdev;
uint64_t old_start, new_start;
struct radeon_fence *fence;
int r, i;
rdev = radeon_get_rdev(bo->bdev);
r = radeon_fence_create(rdev, &fence, radeon_copy_ring_index(rdev));
if (unlikely(r)) {
return r;
}
old_start = old_mem->start << PAGE_SHIFT;
new_start = new_mem->start << PAGE_SHIFT;
switch (old_mem->mem_type) {
case TTM_PL_VRAM:
old_start += rdev->mc.vram_start;
break;
case TTM_PL_TT:
old_start += rdev->mc.gtt_start;
break;
default:
DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
return -EINVAL;
}
switch (new_mem->mem_type) {
case TTM_PL_VRAM:
new_start += rdev->mc.vram_start;
break;
case TTM_PL_TT:
new_start += rdev->mc.gtt_start;
break;
default:
DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
return -EINVAL;
}
if (!rdev->ring[radeon_copy_ring_index(rdev)].ready) {
DRM_ERROR("Trying to move memory with ring turned off.\n");
return -EINVAL;
}
BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0);
/* sync other rings */
if (rdev->family >= CHIP_R600) {
for (i = 0; i < RADEON_NUM_RINGS; ++i) {
/* no need to sync to our own or unused rings */
if (i == radeon_copy_ring_index(rdev) || !rdev->ring[i].ready)
continue;
if (!fence->semaphore) {
r = radeon_semaphore_create(rdev, &fence->semaphore);
/* FIXME: handle semaphore error */
if (r)
continue;
}
r = radeon_ring_lock(rdev, &rdev->ring[i], 3);
/* FIXME: handle ring lock error */
if (r)
continue;
radeon_semaphore_emit_signal(rdev, i, fence->semaphore);
radeon_ring_unlock_commit(rdev, &rdev->ring[i]);
r = radeon_ring_lock(rdev, &rdev->ring[radeon_copy_ring_index(rdev)], 3);
/* FIXME: handle ring lock error */
if (r)
continue;
radeon_semaphore_emit_wait(rdev, radeon_copy_ring_index(rdev), fence->semaphore);
radeon_ring_unlock_commit(rdev, &rdev->ring[radeon_copy_ring_index(rdev)]);
}
}
r = radeon_copy(rdev, old_start, new_start,
new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */
fence);
/* FIXME: handle copy error */
r = ttm_bo_move_accel_cleanup(bo, (void *)fence, NULL,
evict, no_wait_reserve, no_wait_gpu, new_mem);
radeon_fence_unref(&fence);
return r;
}
static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
struct ttm_mem_type_manager *man)
{
struct radeon_device *rdev;
rdev = radeon_get_rdev(bdev);
switch (type) {
case TTM_PL_SYSTEM:
man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
man->available_caching = TTM_PL_MASK_CACHING;
man->default_caching = TTM_PL_FLAG_CACHED;
break;
case TTM_PL_TT:
man->gpu_offset = 0;
man->available_caching = TTM_PL_MASK_CACHING;
man->default_caching = TTM_PL_FLAG_CACHED;
man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
if (!(drm_core_has_AGP(rdev->ddev) && rdev->ddev->agp)) {
DRM_ERROR("AGP is not enabled for memory type %u\n",
(unsigned)type);
return -EINVAL;
}
man->io_offset = rdev->mc.agp_base;
man->io_size = rdev->mc.gtt_size;
man->io_addr = NULL;
if (!rdev->ddev->agp->cant_use_aperture)
man->flags = TTM_MEMTYPE_FLAG_NEEDS_IOREMAP |
TTM_MEMTYPE_FLAG_MAPPABLE;
man->available_caching = TTM_PL_FLAG_UNCACHED |
TTM_PL_FLAG_WC;
man->default_caching = TTM_PL_FLAG_WC;
} else
#endif
{
man->io_offset = 0;
man->io_size = 0;
man->io_addr = NULL;
}
break;
case TTM_PL_VRAM:
man->gpu_offset = 0;
man->flags = TTM_MEMTYPE_FLAG_FIXED |
TTM_MEMTYPE_FLAG_NEEDS_IOREMAP |
TTM_MEMTYPE_FLAG_MAPPABLE;
man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
man->default_caching = TTM_PL_FLAG_WC;
man->io_addr = NULL;
man->io_offset = rdev->mc.aper_base;
man->io_size = rdev->mc.aper_size;
break;
default:
DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
return -EINVAL;
}
return 0;
}
static int radeon_ttm_tt_populate(struct ttm_tt *ttm)
{
struct radeon_device *rdev;
struct radeon_ttm_tt *gtt = (void *)ttm;
unsigned i;
int r, seg;
bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
if (ttm->state != tt_unpopulated)
return 0;
if (slave && ttm->sg) {
#ifdef notyet
drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
gtt->ttm.dma_address, ttm->num_pages);
#endif
ttm->state = tt_unbound;
return 0;
}
rdev = radeon_get_rdev(ttm->bdev);
#if __OS_HAS_AGP
if (rdev->flags & RADEON_IS_AGP) {
return ttm_agp_tt_populate(ttm);
}
#endif
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
return ttm_dma_populate(>t->ttm, rdev->dev);
}
#endif
r = ttm_pool_populate(ttm);
if (r) {
return r;
}
for (i = 0; i < ttm->num_pages; i++) {
gtt->segs[i].ds_addr = VM_PAGE_TO_PHYS(ttm->pages[i]);
gtt->segs[i].ds_len = PAGE_SIZE;
}
if (bus_dmamap_load_raw(rdev->dmat, gtt->map, gtt->segs,
ttm->num_pages,
ttm->num_pages * PAGE_SIZE, 0)) {
ttm_pool_unpopulate(ttm);
return -EFAULT;
}
for (seg = 0, i = 0; seg < gtt->map->dm_nsegs; seg++) {
bus_addr_t addr = gtt->map->dm_segs[seg].ds_addr;
bus_size_t len = gtt->map->dm_segs[seg].ds_len;
while (len > 0) {
gtt->ttm.dma_address[i++] = addr;
addr += PAGE_SIZE;
len -= PAGE_SIZE;
}
}
return 0;
}
