/**
* Called via glUnmapBuffer().
*/
static GLboolean
intel_bufferobj_unmap(GLcontext * ctx,
GLenum target, struct gl_buffer_object *obj)
{
struct intel_context *intel = intel_context(ctx);
struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
assert(intel_obj);
assert(obj->Pointer);
if (intel_obj->sys_buffer != NULL) {
/* always keep the mapping around. */
} else if (intel_obj->range_map_buffer != NULL) {
/* Since we've emitted some blits to buffers that will (likely) be used
* in rendering operations in other cache domains in this batch, emit a
* flush. Once again, we wish for a domain tracker in libdrm to cover
* usage inside of a batchbuffer.
*/
intel_batchbuffer_emit_mi_flush(intel->batch);
free(intel_obj->range_map_buffer);
intel_obj->range_map_buffer = NULL;
} else if (intel_obj->range_map_bo != NULL) {
if (intel_obj->mapped_gtt) {
drm_intel_gem_bo_unmap_gtt(intel_obj->range_map_bo);
} else {
drm_intel_bo_unmap(intel_obj->range_map_bo);
}
intel_emit_linear_blit(intel,
intel_obj->buffer, obj->Offset,
intel_obj->range_map_bo, 0,
obj->Length);
/* Since we've emitted some blits to buffers that will (likely) be used
* in rendering operations in other cache domains in this batch, emit a
* flush. Once again, we wish for a domain tracker in libdrm to cover
* usage inside of a batchbuffer.
*/
intel_batchbuffer_emit_mi_flush(intel->batch);
drm_intel_bo_unreference(intel_obj->range_map_bo);
intel_obj->range_map_bo = NULL;
} else if (intel_obj->buffer != NULL) {
if (intel_obj->mapped_gtt) {
drm_intel_gem_bo_unmap_gtt(intel_obj->buffer);
} else {
drm_intel_bo_unmap(intel_obj->buffer);
}
}
obj->Pointer = NULL;
obj->Offset = 0;
obj->Length = 0;
return GL_TRUE;
}
static void
store_dword_loop(int fd)
{
int i;
int num_rings = gem_get_num_rings(fd);
srandom(0xdeadbeef);
for (i = 0; i < SLOW_QUICK(0x100000, 10); i++) {
int ring = random() % num_rings + 1;
if (ring == I915_EXEC_RENDER) {
BEGIN_BATCH(4, 1);
OUT_BATCH(MI_COND_BATCH_BUFFER_END | MI_DO_COMPARE);
OUT_BATCH(0xffffffff); /* compare dword */
OUT_RELOC(target_buffer, I915_GEM_DOMAIN_RENDER,
I915_GEM_DOMAIN_RENDER, 0);
OUT_BATCH(MI_NOOP);
ADVANCE_BATCH();
} else {
BEGIN_BATCH(4, 1);
OUT_BATCH(MI_FLUSH_DW | 1);
OUT_BATCH(0); /* reserved */
OUT_RELOC(target_buffer, I915_GEM_DOMAIN_RENDER,
I915_GEM_DOMAIN_RENDER, 0);
OUT_BATCH(MI_NOOP | (1<<22) | (0xf));
ADVANCE_BATCH();
}
intel_batchbuffer_flush_on_ring(batch, ring);
}
drm_intel_bo_map(target_buffer, 0);
// map to force waiting on rendering
drm_intel_bo_unmap(target_buffer);
}
static void
dummy_reloc_loop_random_ring(int num_rings)
{
int i;
srandom(0xdeadbeef);
for (i = 0; i < 0x100000; i++) {
int ring = random() % num_rings + 1;
BEGIN_BATCH(4, 1);
if (ring == I915_EXEC_RENDER) {
OUT_BATCH(MI_COND_BATCH_BUFFER_END | MI_DO_COMPARE);
OUT_BATCH(0xffffffff); /* compare dword */
OUT_RELOC(target_buffer, I915_GEM_DOMAIN_RENDER,
I915_GEM_DOMAIN_RENDER, 0);
OUT_BATCH(MI_NOOP);
} else {
OUT_BATCH(MI_FLUSH_DW | 1);
OUT_BATCH(0); /* reserved */
OUT_RELOC(target_buffer, I915_GEM_DOMAIN_RENDER,
I915_GEM_DOMAIN_RENDER, 0);
OUT_BATCH(MI_NOOP | (1<<22) | (0xf));
}
ADVANCE_BATCH();
intel_batchbuffer_flush_on_ring(batch, ring);
drm_intel_bo_map(target_buffer, 0);
// map to force waiting on rendering
drm_intel_bo_unmap(target_buffer);
}
}
void
intel_bo_unmap(struct intel_bo *bo)
{
int err;
err = drm_intel_bo_unmap(gem_bo(bo));
assert(!err);
}
void
intel_upload_finish(struct brw_context *brw)
{
if (!brw->upload.bo)
return;
drm_intel_bo_unmap(brw->upload.bo);
drm_intel_bo_unreference(brw->upload.bo);
brw->upload.bo = NULL;
brw->upload.next_offset = 0;
}
static int
intel_image_write(__DRIimage *image, const void *buf, size_t count)
{
if (image->region->map_refcount)
return -1;
if (!(image->usage & __DRI_IMAGE_USE_WRITE))
return -1;
drm_intel_bo_map(image->region->bo, true);
memcpy(image->region->bo->virtual, buf, count);
drm_intel_bo_unmap(image->region->bo);
return 0;
}
static void scratch_buf_write_to_png(struct igt_buf *buf, const char *filename)
{
cairo_surface_t *surface;
cairo_status_t ret;
drm_intel_bo_map(buf->bo, 0);
surface = cairo_image_surface_create_for_data(buf->bo->virtual,
CAIRO_FORMAT_RGB24,
igt_buf_width(buf),
igt_buf_height(buf),
buf->stride);
ret = cairo_surface_write_to_png(surface, filename);
igt_assert(ret == CAIRO_STATUS_SUCCESS);
cairo_surface_destroy(surface);
drm_intel_bo_unmap(buf->bo);
}
int
drm_intel_bo_get_subdata(drm_intel_bo *bo, unsigned long offset,
unsigned long size, void *data)
{
int ret;
if (bo->bufmgr->bo_get_subdata)
return bo->bufmgr->bo_get_subdata(bo, offset, size, data);
if (size == 0 || data == NULL)
return 0;
ret = drm_intel_bo_map(bo, 0);
if (ret)
return ret;
memcpy(data, (unsigned char *)bo->virtual + offset, size);
drm_intel_bo_unmap(bo);
return 0;
}
/**
* The BufferSubData() driver hook.
*
* Implements glBufferSubData(), which replaces a portion of the data in a
* buffer object.
*
* If the data range specified by (size + offset) extends beyond the end of
* the buffer or if data is NULL, no copy is performed.
*/
static void
brw_buffer_subdata(struct gl_context *ctx,
GLintptrARB offset,
GLsizeiptrARB size,
const GLvoid *data,
struct gl_buffer_object *obj)
{
struct brw_context *brw = brw_context(ctx);
struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
bool busy;
if (size == 0)
return;
assert(intel_obj);
/* See if we can unsynchronized write the data into the user's BO. This
* avoids GPU stalls in unfortunately common user patterns (uploading
* sequentially into a BO, with draw calls in between each upload).
*
* Once we've hit this path, we mark this GL BO as preferring stalling to
* blits, so that we can hopefully hit this path again in the future
* (otherwise, an app that might occasionally stall but mostly not will end
* up with blitting all the time, at the cost of bandwidth)
*/
if (brw->has_llc) {
if (offset + size <= intel_obj->gpu_active_start ||
intel_obj->gpu_active_end <= offset) {
drm_intel_gem_bo_map_unsynchronized(intel_obj->buffer);
memcpy(intel_obj->buffer->virtual + offset, data, size);
drm_intel_bo_unmap(intel_obj->buffer);
if (intel_obj->gpu_active_end > intel_obj->gpu_active_start)
intel_obj->prefer_stall_to_blit = true;
return;
}
}
static void
dummy_reloc_loop(void)
{
int i, j;
for (i = 0; i < 0x800; i++) {
BEGIN_BATCH(8);
OUT_BATCH(XY_SRC_COPY_BLT_CMD |
XY_SRC_COPY_BLT_WRITE_ALPHA |
XY_SRC_COPY_BLT_WRITE_RGB);
OUT_BATCH((3 << 24) | /* 32 bits */
(0xcc << 16) | /* copy ROP */
4*4096);
OUT_BATCH(2048 << 16 | 0);
OUT_BATCH((4096) << 16 | (2048));
OUT_RELOC_FENCED(blt_bo, I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, 0);
OUT_BATCH(0 << 16 | 0);
OUT_BATCH(4*4096);
OUT_RELOC_FENCED(blt_bo, I915_GEM_DOMAIN_RENDER, 0, 0);
ADVANCE_BATCH();
intel_batchbuffer_flush(batch);
BEGIN_BATCH(4);
OUT_BATCH(MI_FLUSH_DW | 1);
OUT_BATCH(0); /* reserved */
OUT_RELOC(target_buffer, I915_GEM_DOMAIN_RENDER,
I915_GEM_DOMAIN_RENDER, 0);
OUT_BATCH(MI_NOOP | (1<<22) | (0xf));
ADVANCE_BATCH();
intel_batchbuffer_flush(batch);
drm_intel_bo_map(target_buffer, 0);
// map to force completion
drm_intel_bo_unmap(target_buffer);
}
}
void
intel_miptree_unmap_raw(struct intel_context *intel,
struct intel_mipmap_tree *mt)
{
drm_intel_bo_unmap(mt->region->bo);
}
/**
* \brief A fast path for glGetTexImage.
*
* \see intel_readpixels_tiled_memcpy()
*/
bool
intel_gettexsubimage_tiled_memcpy(struct gl_context *ctx,
struct gl_texture_image *texImage,
GLint xoffset, GLint yoffset,
GLsizei width, GLsizei height,
GLenum format, GLenum type,
GLvoid *pixels,
const struct gl_pixelstore_attrib *packing)
{
struct brw_context *brw = brw_context(ctx);
struct intel_texture_image *image = intel_texture_image(texImage);
int dst_pitch;
/* The miptree's buffer. */
drm_intel_bo *bo;
int error = 0;
uint32_t cpp;
mem_copy_fn mem_copy = NULL;
/* This fastpath is restricted to specific texture types:
* a 2D BGRA, RGBA, L8 or A8 texture. It could be generalized to support
* more types.
*
* FINISHME: The restrictions below on packing alignment and packing row
* length are likely unneeded now because we calculate the destination stride
* with _mesa_image_row_stride. However, before removing the restrictions
* we need tests.
*/
if (!brw->has_llc ||
!(type == GL_UNSIGNED_BYTE || type == GL_UNSIGNED_INT_8_8_8_8_REV) ||
!(texImage->TexObject->Target == GL_TEXTURE_2D ||
texImage->TexObject->Target == GL_TEXTURE_RECTANGLE) ||
pixels == NULL ||
_mesa_is_bufferobj(packing->BufferObj) ||
packing->Alignment > 4 ||
packing->SkipPixels > 0 ||
packing->SkipRows > 0 ||
(packing->RowLength != 0 && packing->RowLength != width) ||
packing->SwapBytes ||
packing->LsbFirst ||
packing->Invert)
return false;
/* We can't handle copying from RGBX or BGRX because the tiled_memcpy
* function doesn't set the last channel to 1.
*/
if (texImage->TexFormat == MESA_FORMAT_B8G8R8X8_UNORM ||
texImage->TexFormat == MESA_FORMAT_R8G8B8X8_UNORM)
return false;
if (!intel_get_memcpy(texImage->TexFormat, format, type, &mem_copy, &cpp,
INTEL_DOWNLOAD))
return false;
/* If this is a nontrivial texture view, let another path handle it instead. */
if (texImage->TexObject->MinLayer)
return false;
if (!image->mt ||
(image->mt->tiling != I915_TILING_X &&
image->mt->tiling != I915_TILING_Y)) {
/* The algorithm is written only for X- or Y-tiled memory. */
return false;
}
/* Since we are going to write raw data to the miptree, we need to resolve
* any pending fast color clears before we start.
*/
intel_miptree_resolve_color(brw, image->mt);
bo = image->mt->bo;
if (drm_intel_bo_references(brw->batch.bo, bo)) {
perf_debug("Flushing before mapping a referenced bo.\n");
intel_batchbuffer_flush(brw);
}
error = brw_bo_map(brw, bo, false /* write enable */, "miptree");
if (error) {
DBG("%s: failed to map bo\n", __func__);
return false;
}
dst_pitch = _mesa_image_row_stride(packing, width, format, type);
DBG("%s: level=%d x,y=(%d,%d) (w,h)=(%d,%d) format=0x%x type=0x%x "
"mesa_format=0x%x tiling=%d "
"packing=(alignment=%d row_length=%d skip_pixels=%d skip_rows=%d)\n",
__func__, texImage->Level, xoffset, yoffset, width, height,
format, type, texImage->TexFormat, image->mt->tiling,
packing->Alignment, packing->RowLength, packing->SkipPixels,
packing->SkipRows);
int level = texImage->Level + texImage->TexObject->MinLevel;
/* Adjust x and y offset based on miplevel */
xoffset += image->mt->level[level].level_x;
yoffset += image->mt->level[level].level_y;
tiled_to_linear(
xoffset * cpp, (xoffset + width) * cpp,
yoffset, yoffset + height,
pixels - (ptrdiff_t) yoffset * dst_pitch - (ptrdiff_t) xoffset * cpp,
bo->virtual,
dst_pitch, image->mt->pitch,
brw->has_swizzling,
image->mt->tiling,
mem_copy
);
drm_intel_bo_unmap(bo);
return true;
}
void genode_unmap_image(__DRIimage *image)
{
drm_intel_bo_unmap(image->bo);
}
/**
* \brief A fast path for glReadPixels
*
* This fast path is taken when the source format is BGRA, RGBA,
* A or L and when the texture memory is X- or Y-tiled. It downloads
* the source data by directly mapping the memory without a GTT fence.
* This then needs to be de-tiled on the CPU before presenting the data to
* the user in the linear fasion.
*
* This is a performance win over the conventional texture download path.
* In the conventional texture download path, the texture is either mapped
* through the GTT or copied to a linear buffer with the blitter before
* handing off to a software path. This allows us to avoid round-tripping
* through the GPU (in the case where we would be blitting) and do only a
* single copy operation.
*/
static bool
intel_readpixels_tiled_memcpy(struct gl_context * ctx,
GLint xoffset, GLint yoffset,
GLsizei width, GLsizei height,
GLenum format, GLenum type,
GLvoid * pixels,
const struct gl_pixelstore_attrib *pack)
{
struct brw_context *brw = brw_context(ctx);
struct gl_renderbuffer *rb = ctx->ReadBuffer->_ColorReadBuffer;
/* This path supports reading from color buffers only */
if (rb == NULL)
return false;
struct intel_renderbuffer *irb = intel_renderbuffer(rb);
int dst_pitch;
/* The miptree's buffer. */
drm_intel_bo *bo;
int error = 0;
uint32_t cpp;
mem_copy_fn mem_copy = NULL;
/* This fastpath is restricted to specific renderbuffer types:
* a 2D BGRA, RGBA, L8 or A8 texture. It could be generalized to support
* more types.
*/
if (!brw->has_llc ||
!(type == GL_UNSIGNED_BYTE || type == GL_UNSIGNED_INT_8_8_8_8_REV) ||
pixels == NULL ||
_mesa_is_bufferobj(pack->BufferObj) ||
pack->Alignment > 4 ||
pack->SkipPixels > 0 ||
pack->SkipRows > 0 ||
(pack->RowLength != 0 && pack->RowLength != width) ||
pack->SwapBytes ||
pack->LsbFirst ||
pack->Invert)
return false;
/* Only a simple blit, no scale, bias or other mapping. */
if (ctx->_ImageTransferState)
return false;
/* It is possible that the renderbuffer (or underlying texture) is
* multisampled. Since ReadPixels from a multisampled buffer requires a
* multisample resolve, we can't handle this here
*/
if (rb->NumSamples > 1)
return false;
/* We can't handle copying from RGBX or BGRX because the tiled_memcpy
* function doesn't set the last channel to 1. Note this checks BaseFormat
* rather than TexFormat in case the RGBX format is being simulated with an
* RGBA format.
*/
if (rb->_BaseFormat == GL_RGB)
return false;
if (!intel_get_memcpy(rb->Format, format, type, &mem_copy, &cpp))
return false;
if (!irb->mt ||
(irb->mt->tiling != I915_TILING_X &&
irb->mt->tiling != I915_TILING_Y)) {
/* The algorithm is written only for X- or Y-tiled memory. */
return false;
}
/* Since we are going to read raw data to the miptree, we need to resolve
* any pending fast color clears before we start.
*/
intel_miptree_all_slices_resolve_color(brw, irb->mt, 0);
bo = irb->mt->bo;
if (drm_intel_bo_references(brw->batch.bo, bo)) {
perf_debug("Flushing before mapping a referenced bo.\n");
intel_batchbuffer_flush(brw);
}
error = brw_bo_map(brw, bo, false /* write enable */, "miptree");
if (error) {
DBG("%s: failed to map bo\n", __func__);
return false;
}
xoffset += irb->mt->level[irb->mt_level].slice[irb->mt_layer].x_offset;
yoffset += irb->mt->level[irb->mt_level].slice[irb->mt_layer].y_offset;
dst_pitch = _mesa_image_row_stride(pack, width, format, type);
/* For a window-system renderbuffer, the buffer is actually flipped
* vertically, so we need to handle that. Since the detiling function
* can only really work in the forwards direction, we have to be a
* little creative. First, we compute the Y-offset of the first row of
* the renderbuffer (in renderbuffer coordinates). We then match that
* with the last row of the client's data. Finally, we give
* tiled_to_linear a negative pitch so that it walks through the
* client's data backwards as it walks through the renderbufer forwards.
*/
if (rb->Name == 0) {
yoffset = rb->Height - yoffset - height;
pixels += (ptrdiff_t) (height - 1) * dst_pitch;
dst_pitch = -dst_pitch;
}
/* We postponed printing this message until having committed to executing
* the function.
*/
DBG("%s: x,y=(%d,%d) (w,h)=(%d,%d) format=0x%x type=0x%x "
"mesa_format=0x%x tiling=%d "
"pack=(alignment=%d row_length=%d skip_pixels=%d skip_rows=%d)\n",
__func__, xoffset, yoffset, width, height,
format, type, rb->Format, irb->mt->tiling,
pack->Alignment, pack->RowLength, pack->SkipPixels,
pack->SkipRows);
tiled_to_linear(
xoffset * cpp, (xoffset + width) * cpp,
yoffset, yoffset + height,
pixels - (ptrdiff_t) yoffset * dst_pitch - (ptrdiff_t) xoffset * cpp,
bo->virtual + irb->mt->offset,
dst_pitch, irb->mt->pitch,
brw->has_swizzling,
irb->mt->tiling,
mem_copy
);
drm_intel_bo_unmap(bo);
return true;
}
