static void radeonCheckQuery(struct gl_context *ctx, struct gl_query_object *q)
{
radeon_print(RADEON_STATE, RADEON_TRACE, "%s: query id %d\n", __FUNCTION__, q->Id);
#ifdef DRM_RADEON_GEM_BUSY
radeonContextPtr radeon = RADEON_CONTEXT(ctx);
if (radeon->radeonScreen->kernel_mm) {
struct radeon_query_object *query = (struct radeon_query_object *)q;
uint32_t domain;
/* Need to perform a flush, as per ARB_occlusion_query spec */
if (radeon_bo_is_referenced_by_cs(query->bo, radeon->cmdbuf.cs)) {
ctx->Driver.Flush(ctx);
}
if (radeon_bo_is_busy(query->bo, &domain) == 0) {
radeonQueryGetResult(ctx, q);
query->Base.Ready = GL_TRUE;
}
} else {
radeonWaitQuery(ctx, q);
}
#else
radeonWaitQuery(ctx, q);
#endif
}
/* This is for the cache bufmgr. */
static boolean radeon_bomgr_is_buffer_busy(struct pb_manager *_mgr,
struct pb_buffer *_buf)
{
struct radeon_bo *bo = radeon_bo(_buf);
if (radeon_bo_is_referenced_by_any_cs(bo)) {
return TRUE;
}
if (radeon_bo_is_busy((struct pb_buffer*)bo, RADEON_USAGE_READWRITE)) {
return TRUE;
}
return FALSE;
}
static void *radeon_bo_map_internal(struct pb_buffer *_buf,
unsigned flags, void *flush_ctx)
{
struct radeon_bo *bo = radeon_bo(_buf);
struct radeon_drm_cs *cs = flush_ctx;
struct drm_radeon_gem_mmap args = {};
void *ptr;
/* If it's not unsynchronized bo_map, flush CS if needed and then wait. */
if (!(flags & PB_USAGE_UNSYNCHRONIZED)) {
/* DONTBLOCK doesn't make sense with UNSYNCHRONIZED. */
if (flags & PB_USAGE_DONTBLOCK) {
if (!(flags & PB_USAGE_CPU_WRITE)) {
/* Mapping for read.
*
* Since we are mapping for read, we don't need to wait
* if the GPU is using the buffer for read too
* (neither one is changing it).
*
* Only check whether the buffer is being used for write. */
if (radeon_bo_is_referenced_by_cs_for_write(cs, bo)) {
cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC);
return NULL;
}
if (radeon_bo_is_busy((struct pb_buffer*)bo,
RADEON_USAGE_WRITE)) {
return NULL;
}
} else {
if (radeon_bo_is_referenced_by_cs(cs, bo)) {
cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC);
return NULL;
}
if (radeon_bo_is_busy((struct pb_buffer*)bo,
RADEON_USAGE_READWRITE)) {
return NULL;
}
}
} else {
if (!(flags & PB_USAGE_CPU_WRITE)) {
/* Mapping for read.
*
* Since we are mapping for read, we don't need to wait
* if the GPU is using the buffer for read too
* (neither one is changing it).
*
* Only check whether the buffer is being used for write. */
if (radeon_bo_is_referenced_by_cs_for_write(cs, bo)) {
cs->flush_cs(cs->flush_data, 0);
}
radeon_bo_wait((struct pb_buffer*)bo,
RADEON_USAGE_WRITE);
} else {
/* Mapping for write. */
if (radeon_bo_is_referenced_by_cs(cs, bo)) {
cs->flush_cs(cs->flush_data, 0);
} else {
/* Try to avoid busy-waiting in radeon_bo_wait. */
if (p_atomic_read(&bo->num_active_ioctls))
radeon_drm_cs_sync_flush(cs);
}
radeon_bo_wait((struct pb_buffer*)bo, RADEON_USAGE_READWRITE);
}
}
}
/* Return the pointer if it's already mapped. */
if (bo->ptr)
return bo->ptr;
/* Map the buffer. */
pipe_mutex_lock(bo->map_mutex);
/* Return the pointer if it's already mapped (in case of a race). */
if (bo->ptr) {
pipe_mutex_unlock(bo->map_mutex);
return bo->ptr;
}
args.handle = bo->handle;
args.offset = 0;
args.size = (uint64_t)bo->size;
if (drmCommandWriteRead(bo->rws->fd,
DRM_RADEON_GEM_MMAP,
&args,
sizeof(args))) {
pipe_mutex_unlock(bo->map_mutex);
fprintf(stderr, "radeon: gem_mmap failed: %p 0x%08X\n",
bo, bo->handle);
return NULL;
}
ptr = mmap(0, args.size, PROT_READ|PROT_WRITE, MAP_SHARED,
bo->rws->fd, args.addr_ptr);
if (ptr == MAP_FAILED) {
pipe_mutex_unlock(bo->map_mutex);
fprintf(stderr, "radeon: mmap failed, errno: %i\n", errno);
return NULL;
}
bo->ptr = ptr;
pipe_mutex_unlock(bo->map_mutex);
return bo->ptr;
}
static Bool
RADEONDownloadFromScreenCS(PixmapPtr pSrc, int x, int y, int w,
int h, char *dst, int dst_pitch)
{
RINFO_FROM_SCREEN(pSrc->drawable.pScreen);
struct radeon_exa_pixmap_priv *driver_priv;
struct radeon_bo *scratch = NULL;
struct radeon_bo *copy_src;
unsigned size;
uint32_t datatype = 0;
uint32_t src_domain = 0;
uint32_t src_pitch_offset;
unsigned bpp = pSrc->drawable.bitsPerPixel;
uint32_t scratch_pitch = RADEON_ALIGN(w * bpp / 8, 64);
uint32_t copy_pitch;
uint32_t swap = RADEON_HOST_DATA_SWAP_NONE;
int ret;
Bool flush = FALSE;
Bool r;
if (bpp < 8)
return FALSE;
driver_priv = exaGetPixmapDriverPrivate(pSrc);
if (!driver_priv || !driver_priv->bo)
return FALSE;
#if X_BYTE_ORDER == X_BIG_ENDIAN
switch (bpp) {
case 32:
swap = RADEON_HOST_DATA_SWAP_32BIT;
break;
case 16:
swap = RADEON_HOST_DATA_SWAP_16BIT;
break;
}
#endif
/* If we know the BO won't end up in VRAM anyway, don't bother with a scratch */
copy_src = driver_priv->bo;
copy_pitch = pSrc->devKind;
if (!(driver_priv->tiling_flags & (RADEON_TILING_MACRO | RADEON_TILING_MICRO))) {
if (radeon_bo_is_referenced_by_cs(driver_priv->bo, info->cs)) {
src_domain = radeon_bo_get_src_domain(driver_priv->bo);
if ((src_domain & (RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM)) ==
(RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM))
src_domain = 0;
else /* A write may be scheduled */
flush = TRUE;
}
if (!src_domain)
radeon_bo_is_busy(driver_priv->bo, &src_domain);
if (src_domain & ~(uint32_t)RADEON_GEM_DOMAIN_VRAM)
goto copy;
}
size = scratch_pitch * h;
scratch = radeon_bo_open(info->bufmgr, 0, size, 0, RADEON_GEM_DOMAIN_GTT, 0);
if (scratch == NULL) {
goto copy;
}
radeon_cs_space_reset_bos(info->cs);
radeon_add_pixmap(info->cs, pSrc, RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
radeon_cs_space_add_persistent_bo(info->cs, scratch, 0, RADEON_GEM_DOMAIN_GTT);
ret = radeon_cs_space_check(info->cs);
if (ret) {
goto copy;
}
RADEONGetDatatypeBpp(pSrc->drawable.bitsPerPixel, &datatype);
RADEONGetPixmapOffsetPitch(pSrc, &src_pitch_offset);
RADEON_SWITCH_TO_2D();
RADEONBlitChunk(pScrn, driver_priv->bo, scratch, datatype, src_pitch_offset,
scratch_pitch << 16, x, y, 0, 0, w, h,
RADEON_GEM_DOMAIN_VRAM | RADEON_GEM_DOMAIN_GTT,
RADEON_GEM_DOMAIN_GTT);
copy_src = scratch;
copy_pitch = scratch_pitch;
flush = TRUE;
copy:
if (flush)
FLUSH_RING();
ret = radeon_bo_map(copy_src, 0);
if (ret) {
ErrorF("failed to map pixmap: %d\n", ret);
r = FALSE;
goto out;
}
r = TRUE;
w *= bpp / 8;
if (copy_src == driver_priv->bo)
size = y * copy_pitch + x * bpp / 8;
else
size = 0;
while (h--) {
RADEONCopySwap((uint8_t*)dst, copy_src->ptr + size, w, swap);
size += copy_pitch;
dst += dst_pitch;
}
radeon_bo_unmap(copy_src);
out:
if (scratch)
radeon_bo_unref(scratch);
return r;
}
static Bool
RADEONUploadToScreenCS(PixmapPtr pDst, int x, int y, int w, int h,
char *src, int src_pitch)
{
ScreenPtr pScreen = pDst->drawable.pScreen;
RINFO_FROM_SCREEN(pScreen);
struct radeon_exa_pixmap_priv *driver_priv;
struct radeon_bo *scratch = NULL;
struct radeon_bo *copy_dst;
unsigned char *dst;
unsigned size;
uint32_t datatype = 0;
uint32_t dst_domain;
uint32_t dst_pitch_offset;
unsigned bpp = pDst->drawable.bitsPerPixel;
uint32_t scratch_pitch = RADEON_ALIGN(w * bpp / 8, 64);
uint32_t copy_pitch;
uint32_t swap = RADEON_HOST_DATA_SWAP_NONE;
int ret;
Bool flush = TRUE;
Bool r;
int i;
if (bpp < 8)
return FALSE;
driver_priv = exaGetPixmapDriverPrivate(pDst);
if (!driver_priv || !driver_priv->bo)
return FALSE;
#if X_BYTE_ORDER == X_BIG_ENDIAN
switch (bpp) {
case 32:
swap = RADEON_HOST_DATA_SWAP_32BIT;
break;
case 16:
swap = RADEON_HOST_DATA_SWAP_16BIT;
break;
}
#endif
/* If we know the BO won't be busy / in VRAM, don't bother with a scratch */
copy_dst = driver_priv->bo;
copy_pitch = pDst->devKind;
if (!(driver_priv->tiling_flags & (RADEON_TILING_MACRO | RADEON_TILING_MICRO))) {
if (!radeon_bo_is_referenced_by_cs(driver_priv->bo, info->cs)) {
flush = FALSE;
if (!radeon_bo_is_busy(driver_priv->bo, &dst_domain) &&
!(dst_domain & RADEON_GEM_DOMAIN_VRAM))
goto copy;
}
/* use cpu copy for fast fb access */
if (info->is_fast_fb)
goto copy;
}
size = scratch_pitch * h;
scratch = radeon_bo_open(info->bufmgr, 0, size, 0, RADEON_GEM_DOMAIN_GTT, 0);
if (scratch == NULL) {
goto copy;
}
radeon_cs_space_reset_bos(info->cs);
radeon_add_pixmap(info->cs, pDst, 0, RADEON_GEM_DOMAIN_VRAM);
radeon_cs_space_add_persistent_bo(info->cs, scratch, RADEON_GEM_DOMAIN_GTT, 0);
ret = radeon_cs_space_check(info->cs);
if (ret) {
goto copy;
}
copy_dst = scratch;
copy_pitch = scratch_pitch;
flush = FALSE;
copy:
if (flush)
radeon_cs_flush_indirect(pScrn);
ret = radeon_bo_map(copy_dst, 0);
if (ret) {
r = FALSE;
goto out;
}
r = TRUE;
size = w * bpp / 8;
dst = copy_dst->ptr;
if (copy_dst == driver_priv->bo)
dst += y * copy_pitch + x * bpp / 8;
for (i = 0; i < h; i++) {
RADEONCopySwap(dst + i * copy_pitch, (uint8_t*)src, size, swap);
src += src_pitch;
}
radeon_bo_unmap(copy_dst);
if (copy_dst == scratch) {
RADEONGetDatatypeBpp(pDst->drawable.bitsPerPixel, &datatype);
RADEONGetPixmapOffsetPitch(pDst, &dst_pitch_offset);
RADEON_SWITCH_TO_2D();
RADEONBlitChunk(pScrn, scratch, driver_priv->bo, datatype, scratch_pitch << 16,
dst_pitch_offset, 0, 0, x, y, w, h,
RADEON_GEM_DOMAIN_GTT, RADEON_GEM_DOMAIN_VRAM);
}
out:
if (scratch)
radeon_bo_unref(scratch);
return r;
}
