static int r600_cs_set_age(struct radeon_cs_int *csi) /* -------------- */
{
struct r600_cs_manager_legacy *csm = (struct r600_cs_manager_legacy*)csi->csm;
struct r600_cs_reloc_legacy *relocs;
int i;
relocs = (struct r600_cs_reloc_legacy *)csi->relocs;
for (i = 0; i < csi->crelocs; i++) {
radeon_bo_legacy_pending(relocs[i].base.bo, csm->pending_age);
radeon_bo_unref(relocs[i].base.bo);
}
return 0;
}
int radeon_uvd_resume(struct radeon_device *rdev)
{
int r;
if (rdev->uvd.vcpu_bo == NULL)
return -EINVAL;
r = radeon_bo_reserve(rdev->uvd.vcpu_bo, false);
if (r) {
radeon_bo_unref(&rdev->uvd.vcpu_bo);
dev_err(rdev->dev, "(%d) failed to reserve UVD bo\n", r);
return r;
}
/* Have been pin in cpu unmap unpin */
radeon_bo_kunmap(rdev->uvd.vcpu_bo);
radeon_bo_unpin(rdev->uvd.vcpu_bo);
r = radeon_bo_pin(rdev->uvd.vcpu_bo, RADEON_GEM_DOMAIN_VRAM,
&rdev->uvd.gpu_addr);
if (r) {
radeon_bo_unreserve(rdev->uvd.vcpu_bo);
radeon_bo_unref(&rdev->uvd.vcpu_bo);
dev_err(rdev->dev, "(%d) UVD bo pin failed\n", r);
return r;
}
r = radeon_bo_kmap(rdev->uvd.vcpu_bo, &rdev->uvd.cpu_addr);
if (r) {
dev_err(rdev->dev, "(%d) UVD map failed\n", r);
return r;
}
radeon_bo_unreserve(rdev->uvd.vcpu_bo);
return 0;
}
Bool radeon_set_shared_pixmap_backing(PixmapPtr ppix, void *fd_handle,
struct radeon_surface *surface)
{
ScrnInfoPtr pScrn = xf86ScreenToScrn(ppix->drawable.pScreen);
RADEONInfoPtr info = RADEONPTR(pScrn);
struct radeon_bo *bo;
int ihandle = (int)(long)fd_handle;
uint32_t size = ppix->devKind * ppix->drawable.height;
bo = radeon_gem_bo_open_prime(info->bufmgr, ihandle, size);
if (!bo)
return FALSE;
memset(surface, 0, sizeof(struct radeon_surface));
if (info->ChipFamily >= CHIP_FAMILY_R600 && info->surf_man) {
surface->npix_x = ppix->drawable.width;
surface->npix_y = ppix->drawable.height;
surface->npix_z = 1;
surface->blk_w = 1;
surface->blk_h = 1;
surface->blk_d = 1;
surface->array_size = 1;
surface->bpe = ppix->drawable.bitsPerPixel / 8;
surface->nsamples = 1;
/* we are requiring a recent enough libdrm version */
surface->flags |= RADEON_SURF_HAS_TILE_MODE_INDEX;
surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_2D, TYPE);
surface->flags |= RADEON_SURF_SET(RADEON_SURF_MODE_LINEAR, MODE);
if (radeon_surface_best(info->surf_man, surface)) {
return FALSE;
}
if (radeon_surface_init(info->surf_man, surface)) {
return FALSE;
}
/* we have to post hack the surface to reflect the actual size
of the shared pixmap */
surface->level[0].pitch_bytes = ppix->devKind;
surface->level[0].nblk_x = ppix->devKind / surface->bpe;
}
radeon_set_pixmap_bo(ppix, bo);
close(ihandle);
/* we have a reference from the alloc and one from set pixmap bo,
drop one */
radeon_bo_unref(bo);
return TRUE;
}
void radeon_miptree_unreference(radeon_mipmap_tree **ptr)
{
radeon_mipmap_tree *mt = *ptr;
if (!mt)
return;
assert(mt->refcount > 0);
mt->refcount--;
if (!mt->refcount) {
radeon_bo_unref(mt->bo);
free(mt);
}
*ptr = 0;
}
static void
radeon_delete_renderbuffer(struct gl_renderbuffer *rb)
{
struct radeon_renderbuffer *rrb = radeon_renderbuffer(rb);
radeon_print(RADEON_TEXTURE, RADEON_TRACE,
"%s(rb %p, rrb %p) \n",
__func__, rb, rrb);
ASSERT(rrb);
if (rrb && rrb->bo) {
radeon_bo_unref(rrb->bo);
}
free(rrb);
}
/**
* Called via glDeleteBuffersARB().
*/
static void
radeonDeleteBufferObject(struct gl_context * ctx,
struct gl_buffer_object *obj)
{
struct radeon_buffer_object *radeon_obj = get_radeon_buffer_object(obj);
if (obj->Pointer) {
radeon_bo_unmap(radeon_obj->bo);
}
if (radeon_obj->bo) {
radeon_bo_unref(radeon_obj->bo);
}
free(radeon_obj);
}
void radeon_uvd_fini(struct radeon_device *rdev)
{
int r;
if (rdev->uvd.vcpu_bo == NULL)
return;
r = radeon_bo_reserve(rdev->uvd.vcpu_bo, false);
if (!r) {
radeon_bo_kunmap(rdev->uvd.vcpu_bo);
radeon_bo_unpin(rdev->uvd.vcpu_bo);
radeon_bo_unreserve(rdev->uvd.vcpu_bo);
}
radeon_bo_unref(&rdev->uvd.vcpu_bo);
release_firmware(rdev->uvd_fw);
}
/**
* Free memory associated with this texture image.
*/
void radeonFreeTexImageData(GLcontext *ctx, struct gl_texture_image *timage)
{
radeon_texture_image* image = get_radeon_texture_image(timage);
if (image->mt) {
radeon_miptree_unreference(&image->mt);
assert(!image->base.Data);
} else {
_mesa_free_texture_image_data(ctx, timage);
}
if (image->bo) {
radeon_bo_unref(image->bo);
image->bo = NULL;
}
if (timage->Data) {
_mesa_free_texmemory(timage->Data);
timage->Data = NULL;
}
}
void radeon_uvd_fini(struct radeon_device *rdev)
{
int r;
if (rdev->uvd.vcpu_bo == NULL)
return;
r = radeon_bo_reserve(rdev->uvd.vcpu_bo, false);
if (!r) {
radeon_bo_kunmap(rdev->uvd.vcpu_bo);
radeon_bo_unpin(rdev->uvd.vcpu_bo);
radeon_bo_unreserve(rdev->uvd.vcpu_bo);
}
radeon_bo_unref(&rdev->uvd.vcpu_bo);
radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX]);
release_firmware(rdev->uvd_fw);
}
/**
* Free memory associated with this texture image.
*/
void radeonFreeTextureImageBuffer(struct gl_context *ctx, struct gl_texture_image *timage)
{
radeon_texture_image* image = get_radeon_texture_image(timage);
if (image->mt) {
radeon_miptree_unreference(&image->mt);
} else {
_swrast_free_texture_image_buffer(ctx, timage);
}
if (image->bo) {
radeon_bo_unref(image->bo);
image->bo = NULL;
}
if (image->base.Buffer) {
_mesa_align_free(image->base.Buffer);
image->base.Buffer = NULL;
}
free(image->base.ImageOffsets);
image->base.ImageOffsets = NULL;
}
static void
radeon_image_target_renderbuffer_storage(struct gl_context *ctx,
struct gl_renderbuffer *rb,
void *image_handle)
{
radeonContextPtr radeon = RADEON_CONTEXT(ctx);
struct radeon_renderbuffer *rrb;
__DRIscreen *screen;
__DRIimage *image;
screen = radeon->radeonScreen->driScreen;
image = screen->dri2.image->lookupEGLImage(screen, image_handle,
screen->loaderPrivate);
if (image == NULL)
return;
rrb = radeon_renderbuffer(rb);
if (ctx->Driver.Flush)
ctx->Driver.Flush(ctx); /* +r6/r7 */
if (rrb->bo)
radeon_bo_unref(rrb->bo);
rrb->bo = image->bo;
radeon_bo_ref(rrb->bo);
fprintf(stderr, "image->bo: %p, name: %d, rbs: w %d -> p %d\n", image->bo, image->bo->handle,
image->width, image->pitch);
rrb->cpp = image->cpp;
rrb->pitch = image->pitch * image->cpp;
rb->Format = image->format;
rb->InternalFormat = image->internal_format;
rb->Width = image->width;
rb->Height = image->height;
rb->Format = image->format;
rb->DataType = image->data_type;
rb->_BaseFormat = _mesa_base_fbo_format(radeon->glCtx,
image->internal_format);
}
/**
* Allocate space for and store data in a buffer object. Any data that was
* previously stored in the buffer object is lost. If data is NULL,
* memory will be allocated, but no copy will occur.
* Called via ctx->Driver.BufferData().
* \return GL_TRUE for success, GL_FALSE if out of memory
*/
static GLboolean
radeonBufferData(struct gl_context * ctx,
GLenum target,
GLsizeiptrARB size,
const GLvoid * data,
GLenum usage,
struct gl_buffer_object *obj)
{
radeonContextPtr radeon = RADEON_CONTEXT(ctx);
struct radeon_buffer_object *radeon_obj = get_radeon_buffer_object(obj);
radeon_obj->Base.Size = size;
radeon_obj->Base.Usage = usage;
if (radeon_obj->bo != NULL) {
radeon_bo_unref(radeon_obj->bo);
radeon_obj->bo = NULL;
}
if (size != 0) {
radeon_obj->bo = radeon_bo_open(radeon->radeonScreen->bom,
0,
size,
32,
RADEON_GEM_DOMAIN_GTT,
0);
if (!radeon_obj->bo)
return GL_FALSE;
if (data != NULL) {
radeon_bo_map(radeon_obj->bo, GL_TRUE);
memcpy(radeon_obj->bo->ptr, data, size);
radeon_bo_unmap(radeon_obj->bo);
}
}
return GL_TRUE;
}
/**
* radeon_ring_fini - tear down the driver ring struct.
*
* @rdev: radeon_device pointer
* @ring: radeon_ring structure holding ring information
*
* Tear down the driver information for the selected ring (all asics).
*/
void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *ring)
{
int r;
struct radeon_bo *ring_obj;
mutex_lock(&rdev->ring_lock);
ring_obj = ring->ring_obj;
ring->ready = false;
ring->ring = NULL;
ring->ring_obj = NULL;
mutex_unlock(&rdev->ring_lock);
if (ring_obj) {
r = radeon_bo_reserve(ring_obj, false);
if (likely(r == 0)) {
radeon_bo_kunmap(ring_obj);
radeon_bo_unpin(ring_obj);
radeon_bo_unreserve(ring_obj);
}
radeon_bo_unref(&ring_obj);
}
}
void radeon_ttm_fini(struct radeon_device *rdev)
{
int r;
if (!rdev->mman.initialized)
return;
if (rdev->stollen_vga_memory) {
r = radeon_bo_reserve(rdev->stollen_vga_memory, false);
if (r == 0) {
radeon_bo_unpin(rdev->stollen_vga_memory);
radeon_bo_unreserve(rdev->stollen_vga_memory);
}
radeon_bo_unref(&rdev->stollen_vga_memory);
}
ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM);
ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT);
ttm_bo_device_release(&rdev->mman.bdev);
radeon_gart_fini(rdev);
radeon_ttm_global_fini(rdev);
rdev->mman.initialized = false;
DRM_INFO("radeon: ttm finalized\n");
}
void r200FlushElts(struct gl_context *ctx)
{
r200ContextPtr rmesa = R200_CONTEXT(ctx);
int nr, elt_used = rmesa->tcl.elt_used;
radeon_print(RADEON_RENDER, RADEON_VERBOSE, "%s %x %d\n", __FUNCTION__, rmesa->tcl.hw_primitive, elt_used);
assert( rmesa->radeon.dma.flush == r200FlushElts );
rmesa->radeon.dma.flush = NULL;
nr = elt_used / 2;
radeon_bo_unmap(rmesa->radeon.tcl.elt_dma_bo);
r200FireEB(rmesa, nr, rmesa->tcl.hw_primitive);
radeon_bo_unref(rmesa->radeon.tcl.elt_dma_bo);
rmesa->radeon.tcl.elt_dma_bo = NULL;
if (R200_ELT_BUF_SZ > elt_used)
radeonReturnDmaRegion(&rmesa->radeon, R200_ELT_BUF_SZ - elt_used);
}
static int cs_gem_erase(struct radeon_cs_int *cs)
{
struct cs_gem *csg = (struct cs_gem*)cs;
unsigned i;
if (csg->relocs_bo) {
for (i = 0; i < csg->base.crelocs; i++) {
if (csg->relocs_bo[i]) {
/* bo might be referenced from another context so have to use atomic opertions */
atomic_dec((atomic_t *)radeon_gem_get_reloc_in_cs((struct radeon_bo*)csg->relocs_bo[i]), cs->id);
radeon_bo_unref((struct radeon_bo *)csg->relocs_bo[i]);
csg->relocs_bo[i] = NULL;
}
}
}
cs->relocs_total_size = 0;
cs->cdw = 0;
cs->section_ndw = 0;
cs->crelocs = 0;
csg->chunks[0].length_dw = 0;
csg->chunks[1].length_dw = 0;
return 0;
}
static int cs_gem_emit(struct radeon_cs_int *cs)
{
struct cs_gem *csg = (struct cs_gem*)cs;
uint64_t chunk_array[2];
unsigned i;
int r;
while (cs->cdw & 7)
radeon_cs_write_dword((struct radeon_cs *)cs, 0x80000000);
#if CS_BOF_DUMP
cs_gem_dump_bof(cs);
#endif
csg->chunks[0].length_dw = cs->cdw;
chunk_array[0] = (uint64_t)(uintptr_t)&csg->chunks[0];
chunk_array[1] = (uint64_t)(uintptr_t)&csg->chunks[1];
csg->cs.num_chunks = 2;
csg->cs.chunks = (uint64_t)(uintptr_t)chunk_array;
r = drmCommandWriteRead(cs->csm->fd, DRM_RADEON_CS,
&csg->cs, sizeof(struct drm_radeon_cs));
for (i = 0; i < csg->base.crelocs; i++) {
csg->relocs_bo[i]->space_accounted = 0;
/* bo might be referenced from another context so have to use atomic opertions */
atomic_dec((atomic_t *)radeon_gem_get_reloc_in_cs((struct radeon_bo*)csg->relocs_bo[i]), cs->id);
radeon_bo_unref((struct radeon_bo *)csg->relocs_bo[i]);
csg->relocs_bo[i] = NULL;
}
cs->csm->read_used = 0;
cs->csm->vram_write_used = 0;
cs->csm->gart_write_used = 0;
return r;
}
static void radeon_benchmark_move(struct radeon_device *rdev, unsigned size,
unsigned sdomain, unsigned ddomain)
{
struct radeon_bo *dobj = NULL;
struct radeon_bo *sobj = NULL;
uint64_t saddr, daddr;
int r, n;
int time;
n = RADEON_BENCHMARK_ITERATIONS;
r = radeon_bo_create(rdev, size, PAGE_SIZE, true, sdomain, NULL, &sobj);
if (r) {
goto out_cleanup;
}
r = radeon_bo_reserve(sobj, false);
if (unlikely(r != 0))
goto out_cleanup;
r = radeon_bo_pin(sobj, sdomain, &saddr);
radeon_bo_unreserve(sobj);
if (r) {
goto out_cleanup;
}
r = radeon_bo_create(rdev, size, PAGE_SIZE, true, ddomain, NULL, &dobj);
if (r) {
goto out_cleanup;
}
r = radeon_bo_reserve(dobj, false);
if (unlikely(r != 0))
goto out_cleanup;
r = radeon_bo_pin(dobj, ddomain, &daddr);
radeon_bo_unreserve(dobj);
if (r) {
goto out_cleanup;
}
/* r100 doesn't have dma engine so skip the test */
/* also, VRAM-to-VRAM test doesn't make much sense for DMA */
/* skip it as well if domains are the same */
if ((rdev->asic->copy.dma) && (sdomain != ddomain)) {
time = radeon_benchmark_do_move(rdev, size, saddr, daddr,
RADEON_BENCHMARK_COPY_DMA, n);
if (time < 0)
goto out_cleanup;
if (time > 0)
radeon_benchmark_log_results(n, size, time,
sdomain, ddomain, "dma");
}
time = radeon_benchmark_do_move(rdev, size, saddr, daddr,
RADEON_BENCHMARK_COPY_BLIT, n);
if (time < 0)
goto out_cleanup;
if (time > 0)
radeon_benchmark_log_results(n, size, time,
sdomain, ddomain, "blit");
out_cleanup:
if (sobj) {
r = radeon_bo_reserve(sobj, false);
if (likely(r == 0)) {
radeon_bo_unpin(sobj);
radeon_bo_unreserve(sobj);
}
radeon_bo_unref(&sobj);
}
if (dobj) {
r = radeon_bo_reserve(dobj, false);
if (likely(r == 0)) {
radeon_bo_unpin(dobj);
radeon_bo_unreserve(dobj);
}
radeon_bo_unref(&dobj);
}
if (r) {
DRM_ERROR("Error while benchmarking BO move.\n");
}
}
/**
* Called via glRenderbufferStorageEXT() to set the format and allocate
* storage for a user-created renderbuffer.
*/
static GLboolean
radeon_alloc_renderbuffer_storage(struct gl_context * ctx, struct gl_renderbuffer *rb,
GLenum internalFormat,
GLuint width, GLuint height)
{
struct radeon_context *radeon = RADEON_CONTEXT(ctx);
struct radeon_renderbuffer *rrb = radeon_renderbuffer(rb);
GLboolean software_buffer = GL_FALSE;
int cpp;
radeon_print(RADEON_TEXTURE, RADEON_TRACE,
"%s(%p, rb %p) \n",
__func__, ctx, rb);
ASSERT(rb->Name != 0);
switch (internalFormat) {
case GL_R3_G3_B2:
case GL_RGB4:
case GL_RGB5:
rb->Format = _dri_texformat_rgb565;
rb->DataType = GL_UNSIGNED_BYTE;
cpp = 2;
break;
case GL_RGB:
case GL_RGB8:
case GL_RGB10:
case GL_RGB12:
case GL_RGB16:
rb->Format = _dri_texformat_argb8888;
rb->DataType = GL_UNSIGNED_BYTE;
cpp = 4;
break;
case GL_RGBA:
case GL_RGBA2:
case GL_RGBA4:
case GL_RGB5_A1:
case GL_RGBA8:
case GL_RGB10_A2:
case GL_RGBA12:
case GL_RGBA16:
rb->Format = _dri_texformat_argb8888;
rb->DataType = GL_UNSIGNED_BYTE;
cpp = 4;
break;
case GL_STENCIL_INDEX:
case GL_STENCIL_INDEX1_EXT:
case GL_STENCIL_INDEX4_EXT:
case GL_STENCIL_INDEX8_EXT:
case GL_STENCIL_INDEX16_EXT:
/* alloc a depth+stencil buffer */
rb->Format = MESA_FORMAT_S8_Z24;
rb->DataType = GL_UNSIGNED_INT_24_8_EXT;
cpp = 4;
break;
case GL_DEPTH_COMPONENT16:
rb->Format = MESA_FORMAT_Z16;
rb->DataType = GL_UNSIGNED_SHORT;
cpp = 2;
break;
case GL_DEPTH_COMPONENT:
case GL_DEPTH_COMPONENT24:
case GL_DEPTH_COMPONENT32:
rb->Format = MESA_FORMAT_X8_Z24;
rb->DataType = GL_UNSIGNED_INT;
cpp = 4;
break;
case GL_DEPTH_STENCIL_EXT:
case GL_DEPTH24_STENCIL8_EXT:
rb->Format = MESA_FORMAT_S8_Z24;
rb->DataType = GL_UNSIGNED_INT_24_8_EXT;
cpp = 4;
break;
default:
_mesa_problem(ctx,
"Unexpected format in radeon_alloc_renderbuffer_storage");
return GL_FALSE;
}
rb->_BaseFormat = _mesa_base_fbo_format(ctx, internalFormat);
if (ctx->Driver.Flush)
ctx->Driver.Flush(ctx); /* +r6/r7 */
if (rrb->bo)
radeon_bo_unref(rrb->bo);
if (software_buffer) {
return _mesa_soft_renderbuffer_storage(ctx, rb, internalFormat,
width, height);
}
else {
uint32_t size;
uint32_t pitch = ((cpp * width + 63) & ~63) / cpp;
if (RADEON_DEBUG & RADEON_MEMORY)
fprintf(stderr,"Allocating %d x %d radeon RBO (pitch %d)\n", width,
height, pitch);
size = pitch * height * cpp;
rrb->pitch = pitch * cpp;
rrb->cpp = cpp;
rrb->bo = radeon_bo_open(radeon->radeonScreen->bom,
0,
size,
0,
RADEON_GEM_DOMAIN_VRAM,
0);
rb->Width = width;
rb->Height = height;
return GL_TRUE;
}
}
void r300SetTexBuffer2(__DRIcontext *pDRICtx, GLint target, GLint glx_texture_format, __DRIdrawable *dPriv)
{
struct gl_texture_unit *texUnit;
struct gl_texture_object *texObj;
struct gl_texture_image *texImage;
struct radeon_renderbuffer *rb;
radeon_texture_image *rImage;
radeonContextPtr radeon;
r300ContextPtr rmesa;
struct radeon_framebuffer *rfb;
radeonTexObjPtr t;
uint32_t pitch_val;
uint32_t internalFormat, type, format;
type = GL_BGRA;
format = GL_UNSIGNED_BYTE;
internalFormat = (glx_texture_format == GLX_TEXTURE_FORMAT_RGB_EXT ? 3 : 4);
radeon = pDRICtx->driverPrivate;
rmesa = pDRICtx->driverPrivate;
rfb = dPriv->driverPrivate;
texUnit = &radeon->glCtx->Texture.Unit[radeon->glCtx->Texture.CurrentUnit];
texObj = _mesa_select_tex_object(radeon->glCtx, texUnit, target);
texImage = _mesa_get_tex_image(radeon->glCtx, texObj, target, 0);
rImage = get_radeon_texture_image(texImage);
t = radeon_tex_obj(texObj);
if (t == NULL) {
return;
}
radeon_update_renderbuffers(pDRICtx, dPriv);
/* back & depth buffer are useless free them right away */
rb = (void*)rfb->base.Attachment[BUFFER_DEPTH].Renderbuffer;
if (rb && rb->bo) {
radeon_bo_unref(rb->bo);
rb->bo = NULL;
}
rb = (void*)rfb->base.Attachment[BUFFER_BACK_LEFT].Renderbuffer;
if (rb && rb->bo) {
radeon_bo_unref(rb->bo);
rb->bo = NULL;
}
rb = rfb->color_rb[0];
if (rb->bo == NULL) {
/* Failed to BO for the buffer */
return;
}
_mesa_lock_texture(radeon->glCtx, texObj);
if (t->bo) {
radeon_bo_unref(t->bo);
t->bo = NULL;
}
if (rImage->bo) {
radeon_bo_unref(rImage->bo);
rImage->bo = NULL;
}
radeon_miptree_unreference(&t->mt);
radeon_miptree_unreference(&rImage->mt);
_mesa_init_teximage_fields(radeon->glCtx, target, texImage,
rb->base.Width, rb->base.Height, 1, 0, rb->cpp);
texImage->RowStride = rb->pitch / rb->cpp;
rImage->bo = rb->bo;
radeon_bo_ref(rImage->bo);
t->bo = rb->bo;
radeon_bo_ref(t->bo);
t->tile_bits = 0;
t->image_override = GL_TRUE;
t->override_offset = 0;
t->pp_txpitch &= (1 << 13) -1;
pitch_val = rb->pitch;
switch (rb->cpp) {
case 4:
if (glx_texture_format == GLX_TEXTURE_FORMAT_RGB_EXT)
t->pp_txformat = R300_EASY_TX_FORMAT(X, Y, Z, ONE, W8Z8Y8X8);
else
t->pp_txformat = R300_EASY_TX_FORMAT(X, Y, Z, W, W8Z8Y8X8);
t->pp_txfilter |= tx_table[2].filter;
pitch_val /= 4;
break;
case 3:
default:
t->pp_txformat = R300_EASY_TX_FORMAT(X, Y, Z, ONE, W8Z8Y8X8);
t->pp_txfilter |= tx_table[4].filter;
pitch_val /= 4;
break;
case 2:
t->pp_txformat = R300_EASY_TX_FORMAT(X, Y, Z, ONE, Z5Y6X5);
t->pp_txfilter |= tx_table[5].filter;
pitch_val /= 2;
break;
}
pitch_val--;
t->pp_txsize = (((R300_TX_WIDTHMASK_MASK & ((rb->base.Width - 1) << R300_TX_WIDTHMASK_SHIFT)))
| ((R300_TX_HEIGHTMASK_MASK & ((rb->base.Height - 1) << R300_TX_HEIGHTMASK_SHIFT))));
t->pp_txsize |= R300_TX_SIZE_TXPITCH_EN;
t->pp_txpitch |= pitch_val;
if (rmesa->radeon.radeonScreen->chip_family >= CHIP_FAMILY_RV515) {
if (rb->base.Width > 2048)
t->pp_txpitch |= R500_TXWIDTH_BIT11;
else
t->pp_txpitch &= ~R500_TXWIDTH_BIT11;
if (rb->base.Height > 2048)
t->pp_txpitch |= R500_TXHEIGHT_BIT11;
else
t->pp_txpitch &= ~R500_TXHEIGHT_BIT11;
}
t->validated = GL_TRUE;
_mesa_unlock_texture(radeon->glCtx, texObj);
return;
}
static Bool
RADEONUploadToScreenCS(PixmapPtr pDst, int x, int y, int w, int h,
char *src, int src_pitch)
{
ScreenPtr pScreen = pDst->drawable.pScreen;
RINFO_FROM_SCREEN(pScreen);
struct radeon_exa_pixmap_priv *driver_priv;
struct radeon_bo *scratch = NULL;
struct radeon_bo *copy_dst;
unsigned char *dst;
unsigned size;
uint32_t datatype = 0;
uint32_t dst_domain;
uint32_t dst_pitch_offset;
unsigned bpp = pDst->drawable.bitsPerPixel;
uint32_t scratch_pitch = RADEON_ALIGN(w * bpp / 8, 64);
uint32_t copy_pitch;
uint32_t swap = RADEON_HOST_DATA_SWAP_NONE;
int ret;
Bool flush = TRUE;
Bool r;
int i;
if (bpp < 8)
return FALSE;
driver_priv = exaGetPixmapDriverPrivate(pDst);
if (!driver_priv || !driver_priv->bo)
return FALSE;
#if X_BYTE_ORDER == X_BIG_ENDIAN
switch (bpp) {
case 32:
swap = RADEON_HOST_DATA_SWAP_32BIT;
break;
case 16:
swap = RADEON_HOST_DATA_SWAP_16BIT;
break;
}
#endif
/* If we know the BO won't be busy / in VRAM, don't bother with a scratch */
copy_dst = driver_priv->bo;
copy_pitch = pDst->devKind;
if (!(driver_priv->tiling_flags & (RADEON_TILING_MACRO | RADEON_TILING_MICRO))) {
if (!radeon_bo_is_referenced_by_cs(driver_priv->bo, info->cs)) {
flush = FALSE;
if (!radeon_bo_is_busy(driver_priv->bo, &dst_domain) &&
!(dst_domain & RADEON_GEM_DOMAIN_VRAM))
goto copy;
}
/* use cpu copy for fast fb access */
if (info->is_fast_fb)
goto copy;
}
size = scratch_pitch * h;
scratch = radeon_bo_open(info->bufmgr, 0, size, 0, RADEON_GEM_DOMAIN_GTT, 0);
if (scratch == NULL) {
goto copy;
}
radeon_cs_space_reset_bos(info->cs);
radeon_add_pixmap(info->cs, pDst, 0, RADEON_GEM_DOMAIN_VRAM);
radeon_cs_space_add_persistent_bo(info->cs, scratch, RADEON_GEM_DOMAIN_GTT, 0);
ret = radeon_cs_space_check(info->cs);
if (ret) {
goto copy;
}
copy_dst = scratch;
copy_pitch = scratch_pitch;
flush = FALSE;
copy:
if (flush)
radeon_cs_flush_indirect(pScrn);
ret = radeon_bo_map(copy_dst, 0);
if (ret) {
r = FALSE;
goto out;
}
r = TRUE;
size = w * bpp / 8;
dst = copy_dst->ptr;
if (copy_dst == driver_priv->bo)
dst += y * copy_pitch + x * bpp / 8;
for (i = 0; i < h; i++) {
RADEONCopySwap(dst + i * copy_pitch, (uint8_t*)src, size, swap);
src += src_pitch;
}
radeon_bo_unmap(copy_dst);
if (copy_dst == scratch) {
RADEONGetDatatypeBpp(pDst->drawable.bitsPerPixel, &datatype);
RADEONGetPixmapOffsetPitch(pDst, &dst_pitch_offset);
RADEON_SWITCH_TO_2D();
RADEONBlitChunk(pScrn, scratch, driver_priv->bo, datatype, scratch_pitch << 16,
dst_pitch_offset, 0, 0, x, y, w, h,
RADEON_GEM_DOMAIN_GTT, RADEON_GEM_DOMAIN_VRAM);
}
out:
if (scratch)
radeon_bo_unref(scratch);
return r;
}
static GLboolean
do_blit_readpixels(struct gl_context * ctx,
GLint x, GLint y, GLsizei width, GLsizei height,
GLenum format, GLenum type,
const struct gl_pixelstore_attrib *pack, GLvoid * pixels)
{
radeonContextPtr radeon = RADEON_CONTEXT(ctx);
const struct radeon_renderbuffer *rrb = radeon_renderbuffer(ctx->ReadBuffer->_ColorReadBuffer);
const gl_format dst_format = gl_format_and_type_to_mesa_format(format, type);
unsigned dst_rowstride, dst_imagesize, aligned_rowstride, flip_y;
struct radeon_bo *dst_buffer;
GLint dst_x = 0, dst_y = 0;
intptr_t dst_offset;
/* It's not worth if number of pixels to copy is really small */
if (width * height < 100) {
return GL_FALSE;
}
if (dst_format == MESA_FORMAT_NONE ||
!radeon->vtbl.check_blit(dst_format) || !radeon->vtbl.blit) {
return GL_FALSE;
}
if (ctx->_ImageTransferState || ctx->Color._LogicOpEnabled) {
return GL_FALSE;
}
if (pack->SwapBytes || pack->LsbFirst) {
return GL_FALSE;
}
if (pack->RowLength > 0) {
dst_rowstride = pack->RowLength;
} else {
dst_rowstride = width;
}
if (!_mesa_clip_copytexsubimage(ctx, &dst_x, &dst_y, &x, &y, &width, &height)) {
return GL_TRUE;
}
assert(x >= 0 && y >= 0);
aligned_rowstride = get_texture_image_row_stride(radeon, dst_format, dst_rowstride, 0);
dst_rowstride *= _mesa_get_format_bytes(dst_format);
if (_mesa_is_bufferobj(pack->BufferObj) && aligned_rowstride != dst_rowstride)
return GL_FALSE;
dst_imagesize = get_texture_image_size(dst_format,
aligned_rowstride,
height, 1, 0);
if (!_mesa_is_bufferobj(pack->BufferObj))
{
dst_buffer = radeon_bo_open(radeon->radeonScreen->bom, 0, dst_imagesize, 1024, RADEON_GEM_DOMAIN_GTT, 0);
dst_offset = 0;
}
else
{
dst_buffer = get_radeon_buffer_object(pack->BufferObj)->bo;
dst_offset = (intptr_t)pixels;
}
/* Disable source Y flipping for FBOs */
flip_y = (ctx->ReadBuffer->Name == 0);
if (pack->Invert) {
y = rrb->base.Height - height - y;
flip_y = !flip_y;
}
if (radeon->vtbl.blit(ctx,
rrb->bo,
rrb->draw_offset,
rrb->base.Format,
rrb->pitch / rrb->cpp,
rrb->base.Width,
rrb->base.Height,
x,
y,
dst_buffer,
dst_offset,
dst_format,
aligned_rowstride / _mesa_get_format_bytes(dst_format),
width,
height,
0, /* dst_x */
0, /* dst_y */
width,
height,
flip_y))
{
if (!_mesa_is_bufferobj(pack->BufferObj))
{
radeon_bo_map(dst_buffer, 0);
copy_rows(pixels, dst_rowstride, dst_buffer->ptr,
aligned_rowstride, height, dst_rowstride);
radeon_bo_unmap(dst_buffer);
radeon_bo_unref(dst_buffer);
}
return GL_TRUE;
}
if (!_mesa_is_bufferobj(pack->BufferObj))
radeon_bo_unref(dst_buffer);
return GL_FALSE;
}
static Bool
RADEONDownloadFromScreenCS(PixmapPtr pSrc, int x, int y, int w,
int h, char *dst, int dst_pitch)
{
RINFO_FROM_SCREEN(pSrc->drawable.pScreen);
struct radeon_exa_pixmap_priv *driver_priv;
struct radeon_bo *scratch = NULL;
struct radeon_bo *copy_src;
unsigned size;
uint32_t datatype = 0;
uint32_t src_domain = 0;
uint32_t src_pitch_offset;
unsigned bpp = pSrc->drawable.bitsPerPixel;
uint32_t scratch_pitch = RADEON_ALIGN(w * bpp / 8, 64);
uint32_t copy_pitch;
uint32_t swap = RADEON_HOST_DATA_SWAP_NONE;
int ret;
Bool flush = FALSE;
Bool r;
if (bpp < 8)
return FALSE;
driver_priv = exaGetPixmapDriverPrivate(pSrc);
if (!driver_priv || !driver_priv->bo)
return FALSE;
#if X_BYTE_ORDER == X_BIG_ENDIAN
switch (bpp) {
case 32:
swap = RADEON_HOST_DATA_SWAP_32BIT;
break;
case 16:
swap = RADEON_HOST_DATA_SWAP_16BIT;
break;
}
#endif
/* If we know the BO won't end up in VRAM anyway, don't bother with a scratch */
copy_src = driver_priv->bo;
copy_pitch = pSrc->devKind;
if (!(driver_priv->tiling_flags & (RADEON_TILING_MACRO | RADEON_TILING_MICRO))) {
if (radeon_bo_is_referenced_by_cs(driver_priv->bo, info->cs)) {
src_domain = radeon_bo_get_src_domain(driver_priv->bo);
if ((src_domain & (RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM)) ==
(RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM))
src_domain = 0;
else /* A write may be scheduled */
flush = TRUE;
}
if (!src_domain)
radeon_bo_is_busy(driver_priv->bo, &src_domain);
if (src_domain & ~(uint32_t)RADEON_GEM_DOMAIN_VRAM)
goto copy;
}
size = scratch_pitch * h;
scratch = radeon_bo_open(info->bufmgr, 0, size, 0, RADEON_GEM_DOMAIN_GTT, 0);
if (scratch == NULL) {
goto copy;
}
radeon_cs_space_reset_bos(info->cs);
radeon_add_pixmap(info->cs, pSrc, RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
radeon_cs_space_add_persistent_bo(info->cs, scratch, 0, RADEON_GEM_DOMAIN_GTT);
ret = radeon_cs_space_check(info->cs);
if (ret) {
goto copy;
}
RADEONGetDatatypeBpp(pSrc->drawable.bitsPerPixel, &datatype);
RADEONGetPixmapOffsetPitch(pSrc, &src_pitch_offset);
RADEON_SWITCH_TO_2D();
RADEONBlitChunk(pScrn, driver_priv->bo, scratch, datatype, src_pitch_offset,
scratch_pitch << 16, x, y, 0, 0, w, h,
RADEON_GEM_DOMAIN_VRAM | RADEON_GEM_DOMAIN_GTT,
RADEON_GEM_DOMAIN_GTT);
copy_src = scratch;
copy_pitch = scratch_pitch;
flush = TRUE;
copy:
if (flush)
FLUSH_RING();
ret = radeon_bo_map(copy_src, 0);
if (ret) {
ErrorF("failed to map pixmap: %d\n", ret);
r = FALSE;
goto out;
}
r = TRUE;
w *= bpp / 8;
if (copy_src == driver_priv->bo)
size = y * copy_pitch + x * bpp / 8;
else
size = 0;
while (h--) {
RADEONCopySwap((uint8_t*)dst, copy_src->ptr + size, w, swap);
size += copy_pitch;
dst += dst_pitch;
}
radeon_bo_unmap(copy_src);
out:
if (scratch)
radeon_bo_unref(scratch);
return r;
}
void radeon_uvd_fini(struct radeon_device *rdev)
{
radeon_uvd_suspend(rdev);
radeon_bo_unref(&rdev->uvd.vcpu_bo);
}
/* Test BO GTT->VRAM and VRAM->GTT GPU copies across the whole GTT aperture */
void radeon_test_moves(struct radeon_device *rdev)
{
struct radeon_bo *vram_obj = NULL;
struct radeon_bo **gtt_obj = NULL;
struct radeon_fence *fence = NULL;
uint64_t gtt_addr, vram_addr;
unsigned i, n, size;
int r;
size = 1024 * 1024;
/* Number of tests =
* (Total GTT - IB pool - writeback page - ring buffers) / test size
*/
n = rdev->mc.gtt_size - RADEON_IB_POOL_SIZE*64*1024;
for (i = 0; i < RADEON_NUM_RINGS; ++i)
n -= rdev->ring[i].ring_size;
if (rdev->wb.wb_obj)
n -= RADEON_GPU_PAGE_SIZE;
if (rdev->ih.ring_obj)
n -= rdev->ih.ring_size;
n /= size;
gtt_obj = kzalloc(n * sizeof(*gtt_obj), GFP_KERNEL);
if (!gtt_obj) {
DRM_ERROR("Failed to allocate %d pointers\n", n);
r = 1;
goto out_cleanup;
}
r = radeon_bo_create(rdev, size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
&vram_obj);
if (r) {
DRM_ERROR("Failed to create VRAM object\n");
goto out_cleanup;
}
r = radeon_bo_reserve(vram_obj, false);
if (unlikely(r != 0))
goto out_cleanup;
r = radeon_bo_pin(vram_obj, RADEON_GEM_DOMAIN_VRAM, &vram_addr);
if (r) {
DRM_ERROR("Failed to pin VRAM object\n");
goto out_cleanup;
}
for (i = 0; i < n; i++) {
void *gtt_map, *vram_map;
void **gtt_start, **gtt_end;
void **vram_start, **vram_end;
r = radeon_bo_create(rdev, size, PAGE_SIZE, true,
RADEON_GEM_DOMAIN_GTT, gtt_obj + i);
if (r) {
DRM_ERROR("Failed to create GTT object %d\n", i);
goto out_cleanup;
}
r = radeon_bo_reserve(gtt_obj[i], false);
if (unlikely(r != 0))
goto out_cleanup;
r = radeon_bo_pin(gtt_obj[i], RADEON_GEM_DOMAIN_GTT, >t_addr);
if (r) {
DRM_ERROR("Failed to pin GTT object %d\n", i);
goto out_cleanup;
}
r = radeon_bo_kmap(gtt_obj[i], >t_map);
if (r) {
DRM_ERROR("Failed to map GTT object %d\n", i);
goto out_cleanup;
}
for (gtt_start = gtt_map, gtt_end = gtt_map + size;
gtt_start < gtt_end;
gtt_start++)
*gtt_start = gtt_start;
radeon_bo_kunmap(gtt_obj[i]);
r = radeon_fence_create(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX);
if (r) {
DRM_ERROR("Failed to create GTT->VRAM fence %d\n", i);
goto out_cleanup;
}
r = radeon_copy(rdev, gtt_addr, vram_addr, size / RADEON_GPU_PAGE_SIZE, fence);
if (r) {
DRM_ERROR("Failed GTT->VRAM copy %d\n", i);
goto out_cleanup;
}
r = radeon_fence_wait(fence, false);
if (r) {
DRM_ERROR("Failed to wait for GTT->VRAM fence %d\n", i);
goto out_cleanup;
}
radeon_fence_unref(&fence);
r = radeon_bo_kmap(vram_obj, &vram_map);
if (r) {
DRM_ERROR("Failed to map VRAM object after copy %d\n", i);
goto out_cleanup;
}
for (gtt_start = gtt_map, gtt_end = gtt_map + size,
vram_start = vram_map, vram_end = vram_map + size;
vram_start < vram_end;
gtt_start++, vram_start++) {
if (*vram_start != gtt_start) {
DRM_ERROR("Incorrect GTT->VRAM copy %d: Got 0x%p, "
"expected 0x%p (GTT/VRAM offset "
"0x%16llx/0x%16llx)\n",
i, *vram_start, gtt_start,
(unsigned long long)
(gtt_addr - rdev->mc.gtt_start +
(void*)gtt_start - gtt_map),
(unsigned long long)
(vram_addr - rdev->mc.vram_start +
(void*)gtt_start - gtt_map));
radeon_bo_kunmap(vram_obj);
goto out_cleanup;
}
*vram_start = vram_start;
}
radeon_bo_kunmap(vram_obj);
r = radeon_fence_create(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX);
if (r) {
DRM_ERROR("Failed to create VRAM->GTT fence %d\n", i);
goto out_cleanup;
}
r = radeon_copy(rdev, vram_addr, gtt_addr, size / RADEON_GPU_PAGE_SIZE, fence);
if (r) {
DRM_ERROR("Failed VRAM->GTT copy %d\n", i);
goto out_cleanup;
}
r = radeon_fence_wait(fence, false);
if (r) {
DRM_ERROR("Failed to wait for VRAM->GTT fence %d\n", i);
goto out_cleanup;
}
radeon_fence_unref(&fence);
r = radeon_bo_kmap(gtt_obj[i], >t_map);
if (r) {
DRM_ERROR("Failed to map GTT object after copy %d\n", i);
goto out_cleanup;
}
for (gtt_start = gtt_map, gtt_end = gtt_map + size,
vram_start = vram_map, vram_end = vram_map + size;
gtt_start < gtt_end;
gtt_start++, vram_start++) {
if (*gtt_start != vram_start) {
DRM_ERROR("Incorrect VRAM->GTT copy %d: Got 0x%p, "
"expected 0x%p (VRAM/GTT offset "
"0x%16llx/0x%16llx)\n",
i, *gtt_start, vram_start,
(unsigned long long)
(vram_addr - rdev->mc.vram_start +
(void*)vram_start - vram_map),
(unsigned long long)
(gtt_addr - rdev->mc.gtt_start +
(void*)vram_start - vram_map));
radeon_bo_kunmap(gtt_obj[i]);
goto out_cleanup;
}
}
radeon_bo_kunmap(gtt_obj[i]);
DRM_INFO("Tested GTT->VRAM and VRAM->GTT copy for GTT offset 0x%llx\n",
gtt_addr - rdev->mc.gtt_start);
}
out_cleanup:
if (vram_obj) {
if (radeon_bo_is_reserved(vram_obj)) {
radeon_bo_unpin(vram_obj);
radeon_bo_unreserve(vram_obj);
}
radeon_bo_unref(&vram_obj);
}
if (gtt_obj) {
for (i = 0; i < n; i++) {
if (gtt_obj[i]) {
if (radeon_bo_is_reserved(gtt_obj[i])) {
radeon_bo_unpin(gtt_obj[i]);
radeon_bo_unreserve(gtt_obj[i]);
}
radeon_bo_unref(>t_obj[i]);
}
}
kfree(gtt_obj);
}
if (fence) {
radeon_fence_unref(&fence);
}
if (r) {
printk(KERN_WARNING "Error while testing BO move.\n");
}
}
/**
* radeon_vce_init - allocate memory, load vce firmware
*
* @rdev: radeon_device pointer
*
* First step to get VCE online, allocate memory and load the firmware
*/
int radeon_vce_init(struct radeon_device *rdev)
{
static const char *fw_version = "[ATI LIB=VCEFW,";
static const char *fb_version = "[ATI LIB=VCEFWSTATS,";
unsigned long size;
const char *fw_name, *c;
uint8_t start, mid, end;
int i, r;
INIT_DELAYED_WORK(&rdev->vce.idle_work, radeon_vce_idle_work_handler);
switch (rdev->family) {
case CHIP_BONAIRE:
case CHIP_KAVERI:
case CHIP_KABINI:
fw_name = FIRMWARE_BONAIRE;
break;
default:
return -EINVAL;
}
r = request_firmware(&rdev->vce_fw, fw_name, rdev->dev);
if (r) {
dev_err(rdev->dev, "radeon_vce: Can't load firmware \"%s\"\n",
fw_name);
return r;
}
/* search for firmware version */
size = rdev->vce_fw->size - strlen(fw_version) - 9;
c = rdev->vce_fw->data;
for (;size > 0; --size, ++c)
if (strncmp(c, fw_version, strlen(fw_version)) == 0)
break;
if (size == 0)
return -EINVAL;
c += strlen(fw_version);
if (sscanf(c, "%2hhd.%2hhd.%2hhd]", &start, &mid, &end) != 3)
return -EINVAL;
/* search for feedback version */
size = rdev->vce_fw->size - strlen(fb_version) - 3;
c = rdev->vce_fw->data;
for (;size > 0; --size, ++c)
if (strncmp(c, fb_version, strlen(fb_version)) == 0)
break;
if (size == 0)
return -EINVAL;
c += strlen(fb_version);
if (sscanf(c, "%2u]", &rdev->vce.fb_version) != 1)
return -EINVAL;
DRM_INFO("Found VCE firmware/feedback version %hhd.%hhd.%hhd / %d!\n",
start, mid, end, rdev->vce.fb_version);
rdev->vce.fw_version = (start << 24) | (mid << 16) | (end << 8);
/* we can only work with this fw version for now */
if (rdev->vce.fw_version != ((40 << 24) | (2 << 16) | (2 << 8)))
return -EINVAL;
/* allocate firmware, stack and heap BO */
size = RADEON_GPU_PAGE_ALIGN(rdev->vce_fw->size) +
RADEON_VCE_STACK_SIZE + RADEON_VCE_HEAP_SIZE;
r = radeon_bo_create(rdev, size, PAGE_SIZE, true,
RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->vce.vcpu_bo);
if (r) {
dev_err(rdev->dev, "(%d) failed to allocate VCE bo\n", r);
return r;
}
r = radeon_bo_reserve(rdev->vce.vcpu_bo, false);
if (r) {
radeon_bo_unref(&rdev->vce.vcpu_bo);
dev_err(rdev->dev, "(%d) failed to reserve VCE bo\n", r);
return r;
}
r = radeon_bo_pin(rdev->vce.vcpu_bo, RADEON_GEM_DOMAIN_VRAM,
&rdev->vce.gpu_addr);
radeon_bo_unreserve(rdev->vce.vcpu_bo);
if (r) {
radeon_bo_unref(&rdev->vce.vcpu_bo);
dev_err(rdev->dev, "(%d) VCE bo pin failed\n", r);
return r;
}
for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) {
atomic_set(&rdev->vce.handles[i], 0);
rdev->vce.filp[i] = NULL;
}
return 0;
}
/* Test BO GTT->VRAM and VRAM->GTT GPU copies across the whole GTT aperture */
static void radeon_do_test_moves(struct radeon_device *rdev, int flag)
{
struct radeon_bo *vram_obj = NULL;
struct radeon_bo **gtt_obj = NULL;
uint64_t gtt_addr, vram_addr;
unsigned n, size;
int i, r, ring;
switch (flag) {
case RADEON_TEST_COPY_DMA:
ring = radeon_copy_dma_ring_index(rdev);
break;
case RADEON_TEST_COPY_BLIT:
ring = radeon_copy_blit_ring_index(rdev);
break;
default:
DRM_ERROR("Unknown copy method\n");
return;
}
size = 1024 * 1024;
/* Number of tests =
* (Total GTT - IB pool - writeback page - ring buffers) / test size
*/
n = rdev->mc.gtt_size - RADEON_IB_POOL_SIZE*64*1024;
for (i = 0; i < RADEON_NUM_RINGS; ++i)
n -= rdev->ring[i].ring_size;
if (rdev->wb.wb_obj)
n -= RADEON_GPU_PAGE_SIZE;
if (rdev->ih.ring_obj)
n -= rdev->ih.ring_size;
n /= size;
gtt_obj = kzalloc(n * sizeof(*gtt_obj), GFP_KERNEL);
if (!gtt_obj) {
DRM_ERROR("Failed to allocate %d pointers\n", n);
r = 1;
goto out_cleanup;
}
r = radeon_bo_create(rdev, size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
NULL, &vram_obj);
if (r) {
DRM_ERROR("Failed to create VRAM object\n");
goto out_cleanup;
}
r = radeon_bo_reserve(vram_obj, false);
if (unlikely(r != 0))
goto out_unref;
r = radeon_bo_pin(vram_obj, RADEON_GEM_DOMAIN_VRAM, &vram_addr);
if (r) {
DRM_ERROR("Failed to pin VRAM object\n");
goto out_unres;
}
for (i = 0; i < n; i++) {
void *gtt_map, *vram_map;
void **gtt_start, **gtt_end;
void **vram_start, **vram_end;
struct radeon_fence *fence = NULL;
r = radeon_bo_create(rdev, size, PAGE_SIZE, true,
RADEON_GEM_DOMAIN_GTT, NULL, gtt_obj + i);
if (r) {
DRM_ERROR("Failed to create GTT object %d\n", i);
goto out_lclean;
}
r = radeon_bo_reserve(gtt_obj[i], false);
if (unlikely(r != 0))
goto out_lclean_unref;
r = radeon_bo_pin(gtt_obj[i], RADEON_GEM_DOMAIN_GTT, >t_addr);
if (r) {
DRM_ERROR("Failed to pin GTT object %d\n", i);
goto out_lclean_unres;
}
r = radeon_bo_kmap(gtt_obj[i], >t_map);
if (r) {
DRM_ERROR("Failed to map GTT object %d\n", i);
goto out_lclean_unpin;
}
for (gtt_start = gtt_map, gtt_end = gtt_map + size;
gtt_start < gtt_end;
gtt_start++)
*gtt_start = gtt_start;
radeon_bo_kunmap(gtt_obj[i]);
if (ring == R600_RING_TYPE_DMA_INDEX)
r = radeon_copy_dma(rdev, gtt_addr, vram_addr, size / RADEON_GPU_PAGE_SIZE, &fence);
else
r = radeon_copy_blit(rdev, gtt_addr, vram_addr, size / RADEON_GPU_PAGE_SIZE, &fence);
if (r) {
DRM_ERROR("Failed GTT->VRAM copy %d\n", i);
goto out_lclean_unpin;
}
r = radeon_fence_wait(fence, false);
if (r) {
DRM_ERROR("Failed to wait for GTT->VRAM fence %d\n", i);
goto out_lclean_unpin;
}
radeon_fence_unref(&fence);
r = radeon_bo_kmap(vram_obj, &vram_map);
if (r) {
DRM_ERROR("Failed to map VRAM object after copy %d\n", i);
goto out_lclean_unpin;
}
for (gtt_start = gtt_map, gtt_end = gtt_map + size,
vram_start = vram_map, vram_end = vram_map + size;
vram_start < vram_end;
gtt_start++, vram_start++) {
if (*vram_start != gtt_start) {
DRM_ERROR("Incorrect GTT->VRAM copy %d: Got 0x%p, "
"expected 0x%p (GTT/VRAM offset "
"0x%16llx/0x%16llx)\n",
i, *vram_start, gtt_start,
(unsigned long long)
(gtt_addr - rdev->mc.gtt_start +
(void*)gtt_start - gtt_map),
(unsigned long long)
(vram_addr - rdev->mc.vram_start +
(void*)gtt_start - gtt_map));
radeon_bo_kunmap(vram_obj);
goto out_lclean_unpin;
}
*vram_start = vram_start;
}
radeon_bo_kunmap(vram_obj);
if (ring == R600_RING_TYPE_DMA_INDEX)
r = radeon_copy_dma(rdev, vram_addr, gtt_addr, size / RADEON_GPU_PAGE_SIZE, &fence);
else
r = radeon_copy_blit(rdev, vram_addr, gtt_addr, size / RADEON_GPU_PAGE_SIZE, &fence);
if (r) {
DRM_ERROR("Failed VRAM->GTT copy %d\n", i);
goto out_lclean_unpin;
}
r = radeon_fence_wait(fence, false);
if (r) {
DRM_ERROR("Failed to wait for VRAM->GTT fence %d\n", i);
goto out_lclean_unpin;
}
radeon_fence_unref(&fence);
r = radeon_bo_kmap(gtt_obj[i], >t_map);
if (r) {
DRM_ERROR("Failed to map GTT object after copy %d\n", i);
goto out_lclean_unpin;
}
for (gtt_start = gtt_map, gtt_end = gtt_map + size,
vram_start = vram_map, vram_end = vram_map + size;
gtt_start < gtt_end;
gtt_start++, vram_start++) {
if (*gtt_start != vram_start) {
DRM_ERROR("Incorrect VRAM->GTT copy %d: Got 0x%p, "
"expected 0x%p (VRAM/GTT offset "
"0x%16llx/0x%16llx)\n",
i, *gtt_start, vram_start,
(unsigned long long)
(vram_addr - rdev->mc.vram_start +
(void*)vram_start - vram_map),
(unsigned long long)
(gtt_addr - rdev->mc.gtt_start +
(void*)vram_start - vram_map));
radeon_bo_kunmap(gtt_obj[i]);
goto out_lclean_unpin;
}
}
radeon_bo_kunmap(gtt_obj[i]);
DRM_INFO("Tested GTT->VRAM and VRAM->GTT copy for GTT offset 0x%llx\n",
gtt_addr - rdev->mc.gtt_start);
continue;
out_lclean_unpin:
radeon_bo_unpin(gtt_obj[i]);
out_lclean_unres:
radeon_bo_unreserve(gtt_obj[i]);
out_lclean_unref:
radeon_bo_unref(>t_obj[i]);
out_lclean:
for (--i; i >= 0; --i) {
radeon_bo_unpin(gtt_obj[i]);
radeon_bo_unreserve(gtt_obj[i]);
radeon_bo_unref(>t_obj[i]);
}
if (fence)
radeon_fence_unref(&fence);
break;
}
radeon_bo_unpin(vram_obj);
out_unres:
radeon_bo_unreserve(vram_obj);
out_unref:
radeon_bo_unref(&vram_obj);
out_cleanup:
kfree(gtt_obj);
if (r) {
printk(KERN_WARNING "Error while testing BO move.\n");
}
}
int radeon_ttm_init(struct radeon_device *rdev)
{
int r;
r = radeon_ttm_global_init(rdev);
if (r) {
return r;
}
/* No others user of address space so set it to 0 */
r = ttm_bo_device_init(&rdev->mman.bdev,
rdev->mman.bo_global_ref.ref.object,
&radeon_bo_driver, DRM_FILE_PAGE_OFFSET,
rdev->need_dma32);
if (r) {
DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
return r;
}
rdev->mman.bdev.iot = rdev->iot;
rdev->mman.bdev.memt = rdev->memt;
rdev->mman.bdev.dmat = rdev->dmat;
rdev->mman.initialized = true;
r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_VRAM,
rdev->mc.real_vram_size >> PAGE_SHIFT);
if (r) {
DRM_ERROR("Failed initializing VRAM heap.\n");
return r;
}
/* Change the size here instead of the init above so only lpfn is affected */
radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
#ifdef __sparc64__
r = radeon_bo_create(rdev, rdev->fb_offset, PAGE_SIZE, true,
RADEON_GEM_DOMAIN_VRAM,
NULL, &rdev->stollen_vga_memory);
#else
r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true,
RADEON_GEM_DOMAIN_VRAM,
NULL, &rdev->stollen_vga_memory);
#endif
if (r) {
return r;
}
r = radeon_bo_reserve(rdev->stollen_vga_memory, false);
if (r)
return r;
r = radeon_bo_pin(rdev->stollen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL);
radeon_bo_unreserve(rdev->stollen_vga_memory);
if (r) {
radeon_bo_unref(&rdev->stollen_vga_memory);
return r;
}
DRM_INFO("radeon: %uM of VRAM memory ready\n",
(unsigned) (rdev->mc.real_vram_size / (1024 * 1024)));
r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT,
rdev->mc.gtt_size >> PAGE_SHIFT);
if (r) {
DRM_ERROR("Failed initializing GTT heap.\n");
return r;
}
DRM_INFO("radeon: %uM of GTT memory ready.\n",
(unsigned)(rdev->mc.gtt_size / (1024 * 1024)));
#ifdef notyet
rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping;
#endif
r = radeon_ttm_debugfs_init(rdev);
if (r) {
DRM_ERROR("Failed to init debugfs\n");
return r;
}
return 0;
}
int radeon_uvd_init(struct radeon_device *rdev)
{
unsigned long bo_size;
const char *fw_name;
int i, r;
INIT_DELAYED_WORK(&rdev->uvd.idle_work, radeon_uvd_idle_work_handler);
switch (rdev->family) {
case CHIP_RV710:
case CHIP_RV730:
case CHIP_RV740:
fw_name = FIRMWARE_RV710;
break;
case CHIP_CYPRESS:
case CHIP_HEMLOCK:
case CHIP_JUNIPER:
case CHIP_REDWOOD:
case CHIP_CEDAR:
fw_name = FIRMWARE_CYPRESS;
break;
case CHIP_SUMO:
case CHIP_SUMO2:
case CHIP_PALM:
case CHIP_CAYMAN:
case CHIP_BARTS:
case CHIP_TURKS:
case CHIP_CAICOS:
fw_name = FIRMWARE_SUMO;
break;
case CHIP_TAHITI:
case CHIP_VERDE:
case CHIP_PITCAIRN:
case CHIP_ARUBA:
fw_name = FIRMWARE_TAHITI;
break;
case CHIP_BONAIRE:
case CHIP_KABINI:
case CHIP_KAVERI:
fw_name = FIRMWARE_BONAIRE;
break;
default:
return -EINVAL;
}
r = request_firmware(&rdev->uvd_fw, fw_name, rdev->dev);
if (r) {
dev_err(rdev->dev, "radeon_uvd: Can't load firmware \"%s\"\n",
fw_name);
return r;
}
bo_size = RADEON_GPU_PAGE_ALIGN(rdev->uvd_fw->size + 8) +
RADEON_UVD_STACK_SIZE + RADEON_UVD_HEAP_SIZE;
r = radeon_bo_create(rdev, bo_size, PAGE_SIZE, true,
RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->uvd.vcpu_bo);
if (r) {
dev_err(rdev->dev, "(%d) failed to allocate UVD bo\n", r);
return r;
}
r = radeon_bo_reserve(rdev->uvd.vcpu_bo, false);
if (r) {
radeon_bo_unref(&rdev->uvd.vcpu_bo);
dev_err(rdev->dev, "(%d) failed to reserve UVD bo\n", r);
return r;
}
r = radeon_bo_pin(rdev->uvd.vcpu_bo, RADEON_GEM_DOMAIN_VRAM,
&rdev->uvd.gpu_addr);
if (r) {
radeon_bo_unreserve(rdev->uvd.vcpu_bo);
radeon_bo_unref(&rdev->uvd.vcpu_bo);
dev_err(rdev->dev, "(%d) UVD bo pin failed\n", r);
return r;
}
r = radeon_bo_kmap(rdev->uvd.vcpu_bo, &rdev->uvd.cpu_addr);
if (r) {
dev_err(rdev->dev, "(%d) UVD map failed\n", r);
return r;
}
radeon_bo_unreserve(rdev->uvd.vcpu_bo);
for (i = 0; i < RADEON_MAX_UVD_HANDLES; ++i) {
atomic_set(&rdev->uvd.handles[i], 0);
rdev->uvd.filp[i] = NULL;
}
return 0;
}
static void
radeon_destroy_image(__DRIimage *image)
{
radeon_bo_unref(image->bo);
free(image);
}
