static int amdgpu_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(dma_buf->priv); struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); struct ttm_operation_ctx ctx = { true, false }; u32 domain = amdgpu_display_framebuffer_domains(adev); int ret; bool reads = (direction == DMA_BIDIRECTIONAL || direction == DMA_FROM_DEVICE); if (!reads || !(domain & AMDGPU_GEM_DOMAIN_GTT)) return 0; /* move to gtt */ ret = amdgpu_bo_reserve(bo, false); if (unlikely(ret != 0)) return ret; if (!bo->pin_count && (bo->allowed_domains & AMDGPU_GEM_DOMAIN_GTT)) { amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); } amdgpu_bo_unreserve(bo); return ret; }
int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); unsigned asize = amdgpu_bo_size(bo); int ret; if (!vma->vm_file) return -ENODEV; if (adev == NULL) return -ENODEV; /* Check for valid size. */ if (asize < vma->vm_end - vma->vm_start) return -EINVAL; if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) || (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) { return -EPERM; } vma->vm_pgoff += amdgpu_bo_mmap_offset(bo) >> PAGE_SHIFT; /* prime mmap does not need to check access, so allow here */ ret = drm_vma_node_allow(&obj->vma_node, vma->vm_file->private_data); if (ret) return ret; ret = ttm_bo_mmap(vma->vm_file, vma, &adev->mman.bdev); drm_vma_node_revoke(&obj->vma_node, vma->vm_file->private_data); return ret; }
static void dce_virtual_crtc_disable(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); dce_virtual_crtc_dpms(crtc, DRM_MODE_DPMS_OFF); if (crtc->primary->fb) { int r; struct amdgpu_framebuffer *amdgpu_fb; struct amdgpu_bo *abo; amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb); abo = gem_to_amdgpu_bo(amdgpu_fb->obj); r = amdgpu_bo_reserve(abo, true); if (unlikely(r)) DRM_ERROR("failed to reserve abo before unpin\n"); else { amdgpu_bo_unpin(abo); amdgpu_bo_unreserve(abo); } } amdgpu_crtc->pll_id = ATOM_PPLL_INVALID; amdgpu_crtc->encoder = NULL; amdgpu_crtc->connector = NULL; }
int amdgpu_gem_prime_pin(struct drm_gem_object *obj) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); long ret = 0; ret = amdgpu_bo_reserve(bo, false); if (unlikely(ret != 0)) return ret; /* * Wait for all shared fences to complete before we switch to future * use of exclusive fence on this prime shared bo. */ ret = reservation_object_wait_timeout_rcu(bo->tbo.resv, true, false, MAX_SCHEDULE_TIMEOUT); if (unlikely(ret < 0)) { DRM_DEBUG_PRIME("Fence wait failed: %li\n", ret); amdgpu_bo_unreserve(bo); return ret; } /* pin buffer into GTT */ ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT, NULL); if (likely(ret == 0)) bo->prime_shared_count++; amdgpu_bo_unreserve(bo); return ret; }
struct sg_table *amdgpu_gem_prime_get_sg_table(struct drm_gem_object *obj) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); int npages = bo->tbo.num_pages; return drm_prime_pages_to_sg(bo->tbo.ttm->pages, npages); }
bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj) { if (!adev->mode_info.rfbdev) return false; if (robj == gem_to_amdgpu_bo(adev->mode_info.rfbdev->rfb.obj)) return true; return false; }
int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd, struct kgd_dev **dma_buf_kgd, uint64_t *bo_size, void *metadata_buffer, size_t buffer_size, uint32_t *metadata_size, uint32_t *flags) { struct amdgpu_device *adev = (struct amdgpu_device *)kgd; struct dma_buf *dma_buf; struct drm_gem_object *obj; struct amdgpu_bo *bo; uint64_t metadata_flags; int r = -EINVAL; dma_buf = dma_buf_get(dma_buf_fd); if (IS_ERR(dma_buf)) return PTR_ERR(dma_buf); if (dma_buf->ops != &amdgpu_dmabuf_ops) /* Can't handle non-graphics buffers */ goto out_put; obj = dma_buf->priv; if (obj->dev->driver != adev->ddev->driver) /* Can't handle buffers from different drivers */ goto out_put; adev = obj->dev->dev_private; bo = gem_to_amdgpu_bo(obj); if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT))) /* Only VRAM and GTT BOs are supported */ goto out_put; r = 0; if (dma_buf_kgd) *dma_buf_kgd = (struct kgd_dev *)adev; if (bo_size) *bo_size = amdgpu_bo_size(bo); if (metadata_size) *metadata_size = bo->metadata_size; if (metadata_buffer) r = amdgpu_bo_get_metadata(bo, metadata_buffer, buffer_size, metadata_size, &metadata_flags); if (flags) { *flags = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? ALLOC_MEM_FLAGS_VRAM : ALLOC_MEM_FLAGS_GTT; if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) *flags |= ALLOC_MEM_FLAGS_PUBLIC; } out_put: dma_buf_put(dma_buf); return r; }
void amdgpu_gem_object_free(struct drm_gem_object *gobj) { struct amdgpu_bo *robj = gem_to_amdgpu_bo(gobj); if (robj) { if (robj->gem_base.import_attach) drm_prime_gem_destroy(&robj->gem_base, robj->tbo.sg); amdgpu_mn_unregister(robj); amdgpu_bo_unref(&robj); } }
struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev, struct drm_gem_object *gobj, int flags) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj); if (amdgpu_ttm_tt_has_userptr(bo->tbo.ttm)) return ERR_PTR(-EPERM); return drm_gem_prime_export(dev, gobj, flags); }
int amdgpu_fbdev_total_size(struct amdgpu_device *adev) { struct amdgpu_bo *robj; int size = 0; if (!adev->mode_info.rfbdev) return 0; robj = gem_to_amdgpu_bo(adev->mode_info.rfbdev->rfb.obj); size += amdgpu_bo_size(robj); return size; }
void amdgpu_gem_prime_unpin(struct drm_gem_object *obj) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); int ret = 0; ret = amdgpu_bo_reserve(bo, false); if (unlikely(ret != 0)) return; amdgpu_bo_unpin(bo); amdgpu_bo_unreserve(bo); }
void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); int ret; ret = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->dma_buf_vmap); if (ret) return ERR_PTR(ret); return bo->dma_buf_vmap.virtual; }
static void amdgpufb_destroy_pinned_object(struct drm_gem_object *gobj) { struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj); int ret; ret = amdgpu_bo_reserve(abo, true); if (likely(ret == 0)) { amdgpu_bo_kunmap(abo); amdgpu_bo_unpin(abo); amdgpu_bo_unreserve(abo); } drm_gem_object_unreference_unlocked(gobj); }
int amdgpu_gem_prime_pin(struct drm_gem_object *obj) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); int ret = 0; ret = amdgpu_bo_reserve(bo, false); if (unlikely(ret != 0)) return ret; /* pin buffer into GTT */ ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT, NULL); amdgpu_bo_unreserve(bo); return ret; }
void amdgpu_gem_prime_unpin(struct drm_gem_object *obj) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); int ret = 0; ret = amdgpu_bo_reserve(bo, true); if (unlikely(ret != 0)) return; amdgpu_bo_unpin(bo); if (bo->prime_shared_count) bo->prime_shared_count--; amdgpu_bo_unreserve(bo); }
static int amdgpu_gem_map_attach(struct dma_buf *dma_buf, struct device *target_dev, struct dma_buf_attachment *attach) { struct drm_gem_object *obj = dma_buf->priv; struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); long r; r = drm_gem_map_attach(dma_buf, target_dev, attach); if (r) return r; r = amdgpu_bo_reserve(bo, false); if (unlikely(r != 0)) goto error_detach; if (attach->dev->driver != adev->dev->driver) { /* * Wait for all shared fences to complete before we switch to future * use of exclusive fence on this prime shared bo. */ r = reservation_object_wait_timeout_rcu(bo->tbo.resv, true, false, MAX_SCHEDULE_TIMEOUT); if (unlikely(r < 0)) { DRM_DEBUG_PRIME("Fence wait failed: %li\n", r); goto error_unreserve; } } /* pin buffer into GTT */ r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT, NULL); if (r) goto error_unreserve; if (attach->dev->driver != adev->dev->driver) bo->prime_shared_count++; error_unreserve: amdgpu_bo_unreserve(bo); error_detach: if (r) drm_gem_map_detach(dma_buf, attach); return r; }
struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev, struct drm_gem_object *gobj, int flags) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj); struct dma_buf *buf; if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) || bo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) return ERR_PTR(-EPERM); buf = drm_gem_prime_export(dev, gobj, flags); if (!IS_ERR(buf)) buf->file->f_mapping = dev->anon_inode->i_mapping; return buf; }
static void amdgpu_gem_map_detach(struct dma_buf *dma_buf, struct dma_buf_attachment *attach) { struct drm_gem_object *obj = dma_buf->priv; struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); int ret = 0; ret = amdgpu_bo_reserve(bo, true); if (unlikely(ret != 0)) goto error; amdgpu_bo_unpin(bo); if (attach->dev->driver != adev->dev->driver && bo->prime_shared_count) bo->prime_shared_count--; amdgpu_bo_unreserve(bo); error: drm_gem_map_detach(dma_buf, attach); }
static int amdgpu_bo_list_set(struct amdgpu_device *adev, struct drm_file *filp, struct amdgpu_bo_list *list, struct drm_amdgpu_bo_list_entry *info, unsigned num_entries) { struct amdgpu_bo_list_entry *array; struct amdgpu_bo *gds_obj = adev->gds.gds_gfx_bo; struct amdgpu_bo *gws_obj = adev->gds.gws_gfx_bo; struct amdgpu_bo *oa_obj = adev->gds.oa_gfx_bo; bool has_userptr = false; unsigned i; array = drm_malloc_ab(num_entries, sizeof(struct amdgpu_bo_list_entry)); if (!array) return -ENOMEM; memset(array, 0, num_entries * sizeof(struct amdgpu_bo_list_entry)); for (i = 0; i < num_entries; ++i) { struct amdgpu_bo_list_entry *entry = &array[i]; struct drm_gem_object *gobj; gobj = drm_gem_object_lookup(adev->ddev, filp, info[i].bo_handle); if (!gobj) goto error_free; entry->robj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); drm_gem_object_unreference_unlocked(gobj); entry->priority = info[i].bo_priority; entry->prefered_domains = entry->robj->initial_domain; entry->allowed_domains = entry->prefered_domains; if (entry->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) entry->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; if (amdgpu_ttm_tt_has_userptr(entry->robj->tbo.ttm)) { has_userptr = true; entry->prefered_domains = AMDGPU_GEM_DOMAIN_GTT; entry->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; } entry->tv.bo = &entry->robj->tbo; entry->tv.shared = !entry->robj->prime_shared_count; if (entry->prefered_domains == AMDGPU_GEM_DOMAIN_GDS) gds_obj = entry->robj; if (entry->prefered_domains == AMDGPU_GEM_DOMAIN_GWS) gws_obj = entry->robj; if (entry->prefered_domains == AMDGPU_GEM_DOMAIN_OA) oa_obj = entry->robj; trace_amdgpu_bo_list_set(list, entry->robj); } for (i = 0; i < list->num_entries; ++i) amdgpu_bo_unref(&list->array[i].robj); drm_free_large(list->array); list->gds_obj = gds_obj; list->gws_obj = gws_obj; list->oa_obj = oa_obj; list->has_userptr = has_userptr; list->array = array; list->num_entries = num_entries; return 0; error_free: drm_free_large(array); return -ENOENT; }
static int amdgpu_bo_list_set(struct amdgpu_device *adev, struct drm_file *filp, struct amdgpu_bo_list *list, struct drm_amdgpu_bo_list_entry *info, unsigned num_entries) { struct amdgpu_bo_list_entry *array; struct amdgpu_bo *gds_obj = adev->gds.gds_gfx_bo; struct amdgpu_bo *gws_obj = adev->gds.gws_gfx_bo; struct amdgpu_bo *oa_obj = adev->gds.oa_gfx_bo; unsigned last_entry = 0, first_userptr = num_entries; unsigned i; int r; unsigned long total_size = 0; array = kvmalloc_array(num_entries, sizeof(struct amdgpu_bo_list_entry), GFP_KERNEL); if (!array) return -ENOMEM; memset(array, 0, num_entries * sizeof(struct amdgpu_bo_list_entry)); for (i = 0; i < num_entries; ++i) { struct amdgpu_bo_list_entry *entry; struct drm_gem_object *gobj; struct amdgpu_bo *bo; struct mm_struct *usermm; gobj = drm_gem_object_lookup(filp, info[i].bo_handle); if (!gobj) { r = -ENOENT; goto error_free; } bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); drm_gem_object_put_unlocked(gobj); usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); if (usermm) { if (usermm != current->mm) { amdgpu_bo_unref(&bo); r = -EPERM; goto error_free; } entry = &array[--first_userptr]; } else { entry = &array[last_entry++]; } entry->robj = bo; entry->priority = min(info[i].bo_priority, AMDGPU_BO_LIST_MAX_PRIORITY); entry->tv.bo = &entry->robj->tbo; entry->tv.shared = !entry->robj->prime_shared_count; if (entry->robj->preferred_domains == AMDGPU_GEM_DOMAIN_GDS) gds_obj = entry->robj; if (entry->robj->preferred_domains == AMDGPU_GEM_DOMAIN_GWS) gws_obj = entry->robj; if (entry->robj->preferred_domains == AMDGPU_GEM_DOMAIN_OA) oa_obj = entry->robj; total_size += amdgpu_bo_size(entry->robj); trace_amdgpu_bo_list_set(list, entry->robj); } for (i = 0; i < list->num_entries; ++i) amdgpu_bo_unref(&list->array[i].robj); kvfree(list->array); list->gds_obj = gds_obj; list->gws_obj = gws_obj; list->oa_obj = oa_obj; list->first_userptr = first_userptr; list->array = array; list->num_entries = num_entries; trace_amdgpu_cs_bo_status(list->num_entries, total_size); return 0; error_free: while (i--) amdgpu_bo_unref(&array[i].robj); kvfree(array); return r; }
static int amdgpufb_create(struct drm_fb_helper *helper, struct drm_fb_helper_surface_size *sizes) { struct amdgpu_fbdev *rfbdev = (struct amdgpu_fbdev *)helper; struct amdgpu_device *adev = rfbdev->adev; struct fb_info *info; struct drm_framebuffer *fb = NULL; struct drm_mode_fb_cmd2 mode_cmd; struct drm_gem_object *gobj = NULL; struct amdgpu_bo *abo = NULL; int ret; unsigned long tmp; mode_cmd.width = sizes->surface_width; mode_cmd.height = sizes->surface_height; if (sizes->surface_bpp == 24) sizes->surface_bpp = 32; mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp, sizes->surface_depth); ret = amdgpufb_create_pinned_object(rfbdev, &mode_cmd, &gobj); if (ret) { DRM_ERROR("failed to create fbcon object %d\n", ret); return ret; } abo = gem_to_amdgpu_bo(gobj); /* okay we have an object now allocate the framebuffer */ info = drm_fb_helper_alloc_fbi(helper); if (IS_ERR(info)) { ret = PTR_ERR(info); goto out; } info->par = rfbdev; info->skip_vt_switch = true; ret = amdgpu_framebuffer_init(adev->ddev, &rfbdev->rfb, &mode_cmd, gobj); if (ret) { DRM_ERROR("failed to initialize framebuffer %d\n", ret); goto out; } fb = &rfbdev->rfb.base; /* setup helper */ rfbdev->helper.fb = fb; strcpy(info->fix.id, "amdgpudrmfb"); drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth); info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT; info->fbops = &amdgpufb_ops; tmp = amdgpu_bo_gpu_offset(abo) - adev->mc.vram_start; info->fix.smem_start = adev->mc.aper_base + tmp; info->fix.smem_len = amdgpu_bo_size(abo); info->screen_base = abo->kptr; info->screen_size = amdgpu_bo_size(abo); drm_fb_helper_fill_var(info, &rfbdev->helper, sizes->fb_width, sizes->fb_height); /* setup aperture base/size for vesafb takeover */ info->apertures->ranges[0].base = adev->ddev->mode_config.fb_base; info->apertures->ranges[0].size = adev->mc.aper_size; /* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */ if (info->screen_base == NULL) { ret = -ENOSPC; goto out; } DRM_INFO("fb mappable at 0x%lX\n", info->fix.smem_start); DRM_INFO("vram apper at 0x%lX\n", (unsigned long)adev->mc.aper_base); DRM_INFO("size %lu\n", (unsigned long)amdgpu_bo_size(abo)); DRM_INFO("fb depth is %d\n", fb->format->depth); DRM_INFO(" pitch is %d\n", fb->pitches[0]); vga_switcheroo_client_fb_set(adev->ddev->pdev, info); return 0; out: if (abo) { } if (fb && ret) { drm_gem_object_unreference_unlocked(gobj); drm_framebuffer_unregister_private(fb); drm_framebuffer_cleanup(fb); kfree(fb); } return ret; }
static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev, struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **gobj_p) { struct amdgpu_device *adev = rfbdev->adev; struct drm_gem_object *gobj = NULL; struct amdgpu_bo *abo = NULL; bool fb_tiled = false; /* useful for testing */ u32 tiling_flags = 0; int ret; int aligned_size, size; int height = mode_cmd->height; u32 cpp; cpp = drm_format_plane_cpp(mode_cmd->pixel_format, 0); /* need to align pitch with crtc limits */ mode_cmd->pitches[0] = amdgpu_align_pitch(adev, mode_cmd->width, cpp, fb_tiled); height = ALIGN(mode_cmd->height, 8); size = mode_cmd->pitches[0] * height; aligned_size = ALIGN(size, PAGE_SIZE); ret = amdgpu_gem_object_create(adev, aligned_size, 0, AMDGPU_GEM_DOMAIN_VRAM, AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_VRAM_CLEARED, true, &gobj); if (ret) { pr_err("failed to allocate framebuffer (%d)\n", aligned_size); return -ENOMEM; } abo = gem_to_amdgpu_bo(gobj); if (fb_tiled) tiling_flags = AMDGPU_TILING_SET(ARRAY_MODE, GRPH_ARRAY_2D_TILED_THIN1); ret = amdgpu_bo_reserve(abo, false); if (unlikely(ret != 0)) goto out_unref; if (tiling_flags) { ret = amdgpu_bo_set_tiling_flags(abo, tiling_flags); if (ret) dev_err(adev->dev, "FB failed to set tiling flags\n"); } ret = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM, NULL); if (ret) { amdgpu_bo_unreserve(abo); goto out_unref; } ret = amdgpu_bo_kmap(abo, NULL); amdgpu_bo_unreserve(abo); if (ret) { goto out_unref; } *gobj_p = gobj; return 0; out_unref: amdgpufb_destroy_pinned_object(gobj); *gobj_p = NULL; return ret; }
void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); ttm_bo_kunmap(&bo->dma_buf_vmap); }
struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *obj) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); return bo->tbo.resv; }