void r600_dma_copy_buffer(struct r600_context *rctx,
struct pipe_resource *dst,
struct pipe_resource *src,
uint64_t dst_offset,
uint64_t src_offset,
uint64_t size)
{
struct radeon_winsys_cs *cs = rctx->b.dma.cs;
unsigned i, ncopy, csize;
struct r600_resource *rdst = (struct r600_resource*)dst;
struct r600_resource *rsrc = (struct r600_resource*)src;
/* Mark the buffer range of destination as valid (initialized),
* so that transfer_map knows it should wait for the GPU when mapping
* that range. */
util_range_add(&rdst->valid_buffer_range, dst_offset,
dst_offset + size);
size >>= 2; /* convert to dwords */
ncopy = (size / R600_DMA_COPY_MAX_SIZE_DW) + !!(size % R600_DMA_COPY_MAX_SIZE_DW);
r600_need_dma_space(&rctx->b, ncopy * 5, rdst, rsrc);
for (i = 0; i < ncopy; i++) {
csize = size < R600_DMA_COPY_MAX_SIZE_DW ? size : R600_DMA_COPY_MAX_SIZE_DW;
/* emit reloc before writing cs so that cs is always in consistent state */
radeon_add_to_buffer_list(&rctx->b, &rctx->b.dma, rsrc, RADEON_USAGE_READ,
RADEON_PRIO_SDMA_BUFFER);
radeon_add_to_buffer_list(&rctx->b, &rctx->b.dma, rdst, RADEON_USAGE_WRITE,
RADEON_PRIO_SDMA_BUFFER);
radeon_emit(cs, DMA_PACKET(DMA_PACKET_COPY, 0, 0, csize));
radeon_emit(cs, dst_offset & 0xfffffffc);
radeon_emit(cs, src_offset & 0xfffffffc);
radeon_emit(cs, (dst_offset >> 32UL) & 0xff);
radeon_emit(cs, (src_offset >> 32UL) & 0xff);
dst_offset += csize << 2;
src_offset += csize << 2;
size -= csize;
}
r600_dma_emit_wait_idle(&rctx->b);
}
void r600_need_dma_space(struct r600_common_context *ctx, unsigned num_dw,
struct r600_resource *dst, struct r600_resource *src)
{
uint64_t vram = ctx->dma.cs->used_vram;
uint64_t gtt = ctx->dma.cs->used_gart;
if (dst) {
vram += dst->vram_usage;
gtt += dst->gart_usage;
}
if (src) {
vram += src->vram_usage;
gtt += src->gart_usage;
}
/* Flush the GFX IB if DMA depends on it. */
if (radeon_emitted(ctx->gfx.cs, ctx->initial_gfx_cs_size) &&
((dst &&
ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs, dst->buf,
RADEON_USAGE_READWRITE)) ||
(src &&
ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs, src->buf,
RADEON_USAGE_WRITE))))
ctx->gfx.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
/* Flush if there's not enough space, or if the memory usage per IB
* is too large.
*
* IBs using too little memory are limited by the IB submission overhead.
* IBs using too much memory are limited by the kernel/TTM overhead.
* Too long IBs create CPU-GPU pipeline bubbles and add latency.
*
* This heuristic makes sure that DMA requests are executed
* very soon after the call is made and lowers memory usage.
* It improves texture upload performance by keeping the DMA
* engine busy while uploads are being submitted.
*/
num_dw++; /* for emit_wait_idle below */
if (!ctx->ws->cs_check_space(ctx->dma.cs, num_dw) ||
ctx->dma.cs->used_vram + ctx->dma.cs->used_gart > 64 * 1024 * 1024 ||
!radeon_cs_memory_below_limit(ctx->screen, ctx->dma.cs, vram, gtt)) {
ctx->dma.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
assert((num_dw + ctx->dma.cs->current.cdw) <= ctx->dma.cs->current.max_dw);
}
/* Wait for idle if either buffer has been used in the IB before to
* prevent read-after-write hazards.
*/
if ((dst &&
ctx->ws->cs_is_buffer_referenced(ctx->dma.cs, dst->buf,
RADEON_USAGE_READWRITE)) ||
(src &&
ctx->ws->cs_is_buffer_referenced(ctx->dma.cs, src->buf,
RADEON_USAGE_WRITE)))
r600_dma_emit_wait_idle(ctx);
/* If GPUVM is not supported, the CS checker needs 2 entries
* in the buffer list per packet, which has to be done manually.
*/
if (ctx->screen->info.has_virtual_memory) {
if (dst)
radeon_add_to_buffer_list(ctx, &ctx->dma, dst,
RADEON_USAGE_WRITE,
RADEON_PRIO_SDMA_BUFFER);
if (src)
radeon_add_to_buffer_list(ctx, &ctx->dma, src,
RADEON_USAGE_READ,
RADEON_PRIO_SDMA_BUFFER);
}
/* this function is called before all DMA calls, so increment this. */
ctx->num_dma_calls++;
}
