static void * fd_resource_transfer_map(struct pipe_context *pctx, struct pipe_resource *prsc, unsigned level, unsigned usage, const struct pipe_box *box, struct pipe_transfer **pptrans) { struct fd_context *ctx = fd_context(pctx); struct fd_resource *rsc = fd_resource(prsc); struct fd_resource_slice *slice = fd_resource_slice(rsc, level); struct fd_transfer *trans; struct pipe_transfer *ptrans; enum pipe_format format = prsc->format; uint32_t op = 0; uint32_t offset; char *buf; int ret = 0; DBG("prsc=%p, level=%u, usage=%x, box=%dx%d+%d,%d", prsc, level, usage, box->width, box->height, box->x, box->y); ptrans = util_slab_alloc(&ctx->transfer_pool); if (!ptrans) return NULL; /* util_slab_alloc() doesn't zero: */ trans = fd_transfer(ptrans); memset(trans, 0, sizeof(*trans)); pipe_resource_reference(&ptrans->resource, prsc); ptrans->level = level; ptrans->usage = usage; ptrans->box = *box; ptrans->stride = util_format_get_nblocksx(format, slice->pitch) * rsc->cpp; ptrans->layer_stride = rsc->layer_first ? rsc->layer_size : slice->size0; if (usage & PIPE_TRANSFER_READ) op |= DRM_FREEDRENO_PREP_READ; if (usage & PIPE_TRANSFER_WRITE) op |= DRM_FREEDRENO_PREP_WRITE; if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) { realloc_bo(rsc, fd_bo_size(rsc->bo)); if (rsc->stencil) realloc_bo(rsc->stencil, fd_bo_size(rsc->stencil->bo)); fd_invalidate_resource(ctx, prsc); } else if ((usage & PIPE_TRANSFER_WRITE) && prsc->target == PIPE_BUFFER && !util_ranges_intersect(&rsc->valid_buffer_range, box->x, box->x + box->width)) { /* We are trying to write to a previously uninitialized range. No need * to wait. */ } else if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) { /* If the GPU is writing to the resource, or if it is reading from the * resource and we're trying to write to it, flush the renders. */ if (((ptrans->usage & PIPE_TRANSFER_WRITE) && pending(rsc, FD_PENDING_READ | FD_PENDING_WRITE)) || pending(rsc, FD_PENDING_WRITE)) fd_context_render(pctx); /* The GPU keeps track of how the various bo's are being used, and * will wait if necessary for the proper operation to have * completed. */ ret = fd_bo_cpu_prep(rsc->bo, ctx->screen->pipe, op); if (ret) goto fail; } buf = fd_bo_map(rsc->bo); if (!buf) goto fail; offset = slice->offset + box->y / util_format_get_blockheight(format) * ptrans->stride + box->x / util_format_get_blockwidth(format) * rsc->cpp + fd_resource_layer_offset(rsc, slice, box->z); if (prsc->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT || prsc->format == PIPE_FORMAT_X32_S8X24_UINT) { assert(trans->base.box.depth == 1); trans->base.stride = trans->base.box.width * rsc->cpp * 2; trans->staging = malloc(trans->base.stride * trans->base.box.height); if (!trans->staging) goto fail; /* if we're not discarding the whole range (or resource), we must copy * the real data in. */ if (!(usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE | PIPE_TRANSFER_DISCARD_RANGE))) { struct fd_resource_slice *sslice = fd_resource_slice(rsc->stencil, level); void *sbuf = fd_bo_map(rsc->stencil->bo); if (!sbuf) goto fail; float *depth = (float *)(buf + slice->offset + fd_resource_layer_offset(rsc, slice, box->z) + box->y * slice->pitch * 4 + box->x * 4); uint8_t *stencil = sbuf + sslice->offset + fd_resource_layer_offset(rsc->stencil, sslice, box->z) + box->y * sslice->pitch + box->x; if (format != PIPE_FORMAT_X32_S8X24_UINT) util_format_z32_float_s8x24_uint_pack_z_float( trans->staging, trans->base.stride, depth, slice->pitch * 4, box->width, box->height); util_format_z32_float_s8x24_uint_pack_s_8uint( trans->staging, trans->base.stride, stencil, sslice->pitch, box->width, box->height); } buf = trans->staging; offset = 0; } else if (rsc->internal_format != format && util_format_description(format)->layout == UTIL_FORMAT_LAYOUT_RGTC) { assert(trans->base.box.depth == 1); trans->base.stride = util_format_get_stride( format, trans->base.box.width); trans->staging = malloc( util_format_get_2d_size(format, trans->base.stride, trans->base.box.height)); if (!trans->staging) goto fail; /* if we're not discarding the whole range (or resource), we must copy * the real data in. */ if (!(usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE | PIPE_TRANSFER_DISCARD_RANGE))) { uint8_t *rgba8 = (uint8_t *)buf + slice->offset + fd_resource_layer_offset(rsc, slice, box->z) + box->y * slice->pitch * rsc->cpp + box->x * rsc->cpp; switch (format) { case PIPE_FORMAT_RGTC1_UNORM: case PIPE_FORMAT_RGTC1_SNORM: case PIPE_FORMAT_LATC1_UNORM: case PIPE_FORMAT_LATC1_SNORM: util_format_rgtc1_unorm_pack_rgba_8unorm( trans->staging, trans->base.stride, rgba8, slice->pitch * rsc->cpp, box->width, box->height); break; case PIPE_FORMAT_RGTC2_UNORM: case PIPE_FORMAT_RGTC2_SNORM: case PIPE_FORMAT_LATC2_UNORM: case PIPE_FORMAT_LATC2_SNORM: util_format_rgtc2_unorm_pack_rgba_8unorm( trans->staging, trans->base.stride, rgba8, slice->pitch * rsc->cpp, box->width, box->height); break; default: assert(!"Unexpected format"); break; } } buf = trans->staging; offset = 0; } *pptrans = ptrans; return buf + offset; fail: fd_resource_transfer_unmap(pctx, ptrans); return NULL; }
/* In the case of transfer_map of a multi-sample resource, call back into * pctx->transfer_map() to map the staging resource, to handle cases of * MSAA + separate_z32s8 or fake_rgtc */ static void * transfer_map_msaa(struct pipe_context *pctx, struct pipe_resource *prsc, unsigned level, unsigned usage, const struct pipe_box *box, struct pipe_transfer **pptrans) { struct pipe_screen *pscreen = pctx->screen; struct u_transfer *trans = calloc(1, sizeof(*trans)); if (!trans) return NULL; struct pipe_transfer *ptrans = &trans->base; pipe_resource_reference(&ptrans->resource, prsc); ptrans->level = level; ptrans->usage = usage; ptrans->box = *box; struct pipe_resource tmpl = { .target = prsc->target, .format = prsc->format, .width0 = box->width, .height0 = box->height, .depth0 = 1, .array_size = 1, }; trans->ss = pscreen->resource_create(pscreen, &tmpl); if (!trans->ss) { free(trans); return NULL; } if (needs_pack(usage)) { struct pipe_blit_info blit; memset(&blit, 0, sizeof(blit)); blit.src.resource = ptrans->resource; blit.src.format = ptrans->resource->format; blit.src.level = ptrans->level; blit.src.box = *box; blit.dst.resource = trans->ss; blit.dst.format = trans->ss->format; blit.dst.box.width = box->width; blit.dst.box.height = box->height; blit.dst.box.depth = 1; blit.mask = util_format_get_mask(prsc->format); blit.filter = PIPE_TEX_FILTER_NEAREST; pctx->blit(pctx, &blit); } struct pipe_box map_box = *box; map_box.x = 0; map_box.y = 0; void *ss_map = pctx->transfer_map(pctx, trans->ss, 0, usage, &map_box, &trans->trans); if (!ss_map) { free(trans); return NULL; } ptrans->stride = trans->trans->stride; *pptrans = ptrans; return ss_map; } void * u_transfer_helper_transfer_map(struct pipe_context *pctx, struct pipe_resource *prsc, unsigned level, unsigned usage, const struct pipe_box *box, struct pipe_transfer **pptrans) { struct u_transfer_helper *helper = pctx->screen->transfer_helper; struct u_transfer *trans; struct pipe_transfer *ptrans; enum pipe_format format = prsc->format; unsigned width = box->width; unsigned height = box->height; if (!handle_transfer(prsc)) return helper->vtbl->transfer_map(pctx, prsc, level, usage, box, pptrans); if (helper->msaa_map && (prsc->nr_samples > 1)) return transfer_map_msaa(pctx, prsc, level, usage, box, pptrans); debug_assert(box->depth == 1); trans = calloc(1, sizeof(*trans)); if (!trans) return NULL; ptrans = &trans->base; pipe_resource_reference(&ptrans->resource, prsc); ptrans->level = level; ptrans->usage = usage; ptrans->box = *box; ptrans->stride = util_format_get_stride(format, box->width); ptrans->layer_stride = ptrans->stride * box->height; trans->staging = malloc(ptrans->layer_stride); if (!trans->staging) goto fail; trans->ptr = helper->vtbl->transfer_map(pctx, prsc, level, usage, box, &trans->trans); if (!trans->ptr) goto fail; if (prsc->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) { struct pipe_resource *stencil = helper->vtbl->get_stencil(prsc); trans->ptr2 = helper->vtbl->transfer_map(pctx, stencil, level, usage, box, &trans->trans2); if (needs_pack(usage)) { util_format_z32_float_s8x24_uint_pack_z_float(trans->staging, ptrans->stride, trans->ptr, trans->trans->stride, width, height); util_format_z32_float_s8x24_uint_pack_s_8uint(trans->staging, ptrans->stride, trans->ptr2, trans->trans2->stride, width, height); } } else if (needs_pack(usage) && util_format_description(prsc->format)->layout == UTIL_FORMAT_LAYOUT_RGTC) { switch (prsc->format) { case PIPE_FORMAT_RGTC1_UNORM: case PIPE_FORMAT_RGTC1_SNORM: case PIPE_FORMAT_LATC1_UNORM: case PIPE_FORMAT_LATC1_SNORM: util_format_rgtc1_unorm_pack_rgba_8unorm(trans->staging, ptrans->stride, trans->ptr, trans->trans->stride, width, height); break; case PIPE_FORMAT_RGTC2_UNORM: case PIPE_FORMAT_RGTC2_SNORM: case PIPE_FORMAT_LATC2_UNORM: case PIPE_FORMAT_LATC2_SNORM: util_format_rgtc2_unorm_pack_rgba_8unorm(trans->staging, ptrans->stride, trans->ptr, trans->trans->stride, width, height); break; default: assert(!"Unexpected format"); break; } } else { unreachable("bleh"); } *pptrans = ptrans; return trans->staging; fail: if (trans->trans) helper->vtbl->transfer_unmap(pctx, trans->trans); if (trans->trans2) helper->vtbl->transfer_unmap(pctx, trans->trans2); pipe_resource_reference(&ptrans->resource, NULL); free(trans->staging); free(trans); return NULL; } static void flush_region(struct pipe_context *pctx, struct pipe_transfer *ptrans, const struct pipe_box *box) { struct u_transfer_helper *helper = pctx->screen->transfer_helper; struct u_transfer *trans = u_transfer(ptrans); enum pipe_format iformat, format = ptrans->resource->format; unsigned width = box->width; unsigned height = box->height; void *src, *dst; if (!(ptrans->usage & PIPE_TRANSFER_WRITE)) return; if (trans->ss) { struct pipe_blit_info blit; memset(&blit, 0, sizeof(blit)); blit.src.resource = trans->ss; blit.src.format = trans->ss->format; blit.src.box = *box; blit.dst.resource = ptrans->resource; blit.dst.format = ptrans->resource->format; blit.dst.level = ptrans->level; u_box_2d(ptrans->box.x + box->x, ptrans->box.y + box->y, box->width, box->height, &blit.dst.box); blit.mask = util_format_get_mask(ptrans->resource->format); blit.filter = PIPE_TEX_FILTER_NEAREST; pctx->blit(pctx, &blit); return; } iformat = helper->vtbl->get_internal_format(ptrans->resource); src = (uint8_t *)trans->staging + (box->y * ptrans->stride) + (box->x * util_format_get_blocksize(format)); dst = (uint8_t *)trans->ptr + (box->y * trans->trans->stride) + (box->x * util_format_get_blocksize(iformat)); switch (format) { case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: util_format_z32_float_s8x24_uint_unpack_z_float(dst, trans->trans->stride, src, ptrans->stride, width, height); /* fallthru */ case PIPE_FORMAT_X32_S8X24_UINT: dst = (uint8_t *)trans->ptr2 + (box->y * trans->trans2->stride) + (box->x * util_format_get_blocksize(PIPE_FORMAT_S8_UINT)); util_format_z32_float_s8x24_uint_unpack_s_8uint(dst, trans->trans2->stride, src, ptrans->stride, width, height); break; case PIPE_FORMAT_RGTC1_UNORM: case PIPE_FORMAT_RGTC1_SNORM: case PIPE_FORMAT_LATC1_UNORM: case PIPE_FORMAT_LATC1_SNORM: util_format_rgtc1_unorm_unpack_rgba_8unorm(dst, trans->trans->stride, src, ptrans->stride, width, height); break; case PIPE_FORMAT_RGTC2_UNORM: case PIPE_FORMAT_RGTC2_SNORM: case PIPE_FORMAT_LATC2_UNORM: case PIPE_FORMAT_LATC2_SNORM: util_format_rgtc2_unorm_unpack_rgba_8unorm(dst, trans->trans->stride, src, ptrans->stride, width, height); break; default: assert(!"Unexpected staging transfer type"); break; } } void u_transfer_helper_transfer_flush_region(struct pipe_context *pctx, struct pipe_transfer *ptrans, const struct pipe_box *box) { struct u_transfer_helper *helper = pctx->screen->transfer_helper; if (handle_transfer(ptrans->resource)) { struct u_transfer *trans = u_transfer(ptrans); flush_region(pctx, ptrans, box); /* handle MSAA case, since there could be multiple levels of * wrapped transfer, call pctx->transfer_flush_region() * instead of helper->vtbl->transfer_flush_region() */ if (trans->ss) { pctx->transfer_flush_region(pctx, trans->trans, box); return; } helper->vtbl->transfer_flush_region(pctx, trans->trans, box); if (trans->trans2) helper->vtbl->transfer_flush_region(pctx, trans->trans2, box); } else { helper->vtbl->transfer_flush_region(pctx, ptrans, box); } } void u_transfer_helper_transfer_unmap(struct pipe_context *pctx, struct pipe_transfer *ptrans) { struct u_transfer_helper *helper = pctx->screen->transfer_helper; if (handle_transfer(ptrans->resource)) { struct u_transfer *trans = u_transfer(ptrans); if (!(ptrans->usage & PIPE_TRANSFER_FLUSH_EXPLICIT)) { struct pipe_box box; u_box_2d(0, 0, ptrans->box.width, ptrans->box.height, &box); flush_region(pctx, ptrans, &box); } /* in MSAA case, there could be multiple levels of wrapping * so don't call helper->vtbl->transfer_unmap() directly */ if (trans->ss) { pctx->transfer_unmap(pctx, trans->trans); pipe_resource_reference(&trans->ss, NULL); } else { helper->vtbl->transfer_unmap(pctx, trans->trans); if (trans->trans2) helper->vtbl->transfer_unmap(pctx, trans->trans2); } free(trans); } else { helper->vtbl->transfer_unmap(pctx, ptrans); } } struct u_transfer_helper * u_transfer_helper_create(const struct u_transfer_vtbl *vtbl, bool separate_z32s8, bool fake_rgtc, bool msaa_map) { struct u_transfer_helper *helper = calloc(1, sizeof(*helper)); helper->vtbl = vtbl; helper->separate_z32s8 = separate_z32s8; helper->fake_rgtc = fake_rgtc; helper->msaa_map = msaa_map; return helper; } void u_transfer_helper_destroy(struct u_transfer_helper *helper) { free(helper); }