static inline bool
nouveau_buffer_allocate(struct nouveau_screen *screen,
struct nv04_resource *buf, unsigned domain)
{
uint32_t size = align(buf->base.width0, 0x100);
if (domain == NOUVEAU_BO_VRAM) {
buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
&buf->bo, &buf->offset);
if (!buf->bo)
return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_vid, buf->base.width0);
} else
if (domain == NOUVEAU_BO_GART) {
buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
&buf->bo, &buf->offset);
if (!buf->bo)
return false;
NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_sys, buf->base.width0);
} else {
assert(domain == 0);
if (!nouveau_buffer_malloc(buf))
return false;
}
buf->domain = domain;
if (buf->bo)
buf->address = buf->bo->offset + buf->offset;
util_range_set_empty(&buf->valid_buffer_range);
return true;
}
/* Invalidate underlying buffer storage, reset fences, reallocate to non-busy
* buffer.
*/
void
nouveau_buffer_invalidate(struct pipe_context *pipe,
struct pipe_resource *resource)
{
struct nouveau_context *nv = nouveau_context(pipe);
struct nv04_resource *buf = nv04_resource(resource);
int ref = buf->base.reference.count - 1;
/* Shared buffers shouldn't get reallocated */
if (unlikely(buf->base.bind & PIPE_BIND_SHARED))
return;
/* We can't touch persistent/coherent buffers */
if (buf->base.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
PIPE_RESOURCE_FLAG_MAP_COHERENT))
return;
/* If the buffer is sub-allocated and not currently being written, just
* wipe the valid buffer range. Otherwise we have to create fresh
* storage. (We don't keep track of fences for non-sub-allocated BO's.)
*/
if (buf->mm && !nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE)) {
util_range_set_empty(&buf->valid_buffer_range);
} else {
nouveau_buffer_reallocate(nv->screen, buf, buf->domain);
if (ref > 0) /* any references inside context possible ? */
nv->invalidate_resource_storage(nv, &buf->base, ref);
}
}
bool r600_init_resource(struct r600_common_screen *rscreen,
struct r600_resource *res,
unsigned size, unsigned alignment,
bool use_reusable_pool, unsigned usage)
{
uint32_t initial_domain, domains;
switch(usage) {
case PIPE_USAGE_STAGING:
/* Staging resources participate in transfers, i.e. are used
* for uploads and downloads from regular resources.
* We generate them internally for some transfers.
*/
initial_domain = RADEON_DOMAIN_GTT;
domains = RADEON_DOMAIN_GTT;
break;
case PIPE_USAGE_DYNAMIC:
case PIPE_USAGE_STREAM:
/* Default to GTT, but allow the memory manager to move it to VRAM. */
initial_domain = RADEON_DOMAIN_GTT;
domains = RADEON_DOMAIN_GTT | RADEON_DOMAIN_VRAM;
break;
case PIPE_USAGE_DEFAULT:
case PIPE_USAGE_STATIC:
case PIPE_USAGE_IMMUTABLE:
default:
/* Don't list GTT here, because the memory manager would put some
* resources to GTT no matter what the initial domain is.
* Not listing GTT in the domains improves performance a lot. */
initial_domain = RADEON_DOMAIN_VRAM;
domains = RADEON_DOMAIN_VRAM;
break;
}
res->buf = rscreen->ws->buffer_create(rscreen->ws, size, alignment,
use_reusable_pool,
initial_domain);
if (!res->buf) {
return false;
}
res->cs_buf = rscreen->ws->buffer_get_cs_handle(res->buf);
res->domains = domains;
util_range_set_empty(&res->valid_buffer_range);
if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) {
fprintf(stderr, "VM start=0x%"PRIu64" end=0x%"PRIu64" | Buffer %u bytes\n",
r600_resource_va(&rscreen->b, &res->b.b),
r600_resource_va(&rscreen->b, &res->b.b) + res->buf->size,
res->buf->size);
}
return true;
}
bool r600_init_resource(struct r600_common_screen *rscreen,
struct r600_resource *res,
unsigned size, unsigned alignment,
bool use_reusable_pool)
{
struct r600_texture *rtex = (struct r600_texture*)res;
switch (res->b.b.usage) {
case PIPE_USAGE_STAGING:
case PIPE_USAGE_DYNAMIC:
case PIPE_USAGE_STREAM:
/* Transfers are likely to occur more often with these resources. */
res->domains = RADEON_DOMAIN_GTT;
break;
case PIPE_USAGE_DEFAULT:
case PIPE_USAGE_IMMUTABLE:
default:
/* Not listing GTT here improves performance in some apps. */
res->domains = RADEON_DOMAIN_VRAM;
break;
}
/* Tiled textures are unmappable. Always put them in VRAM. */
if (res->b.b.target != PIPE_BUFFER &&
rtex->surface.level[0].mode >= RADEON_SURF_MODE_1D) {
res->domains = RADEON_DOMAIN_VRAM;
}
/* Allocate the resource. */
res->buf = rscreen->ws->buffer_create(rscreen->ws, size, alignment,
use_reusable_pool,
res->domains);
if (!res->buf) {
return false;
}
res->cs_buf = rscreen->ws->buffer_get_cs_handle(res->buf);
util_range_set_empty(&res->valid_buffer_range);
if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) {
fprintf(stderr, "VM start=0x%"PRIu64" end=0x%"PRIu64" | Buffer %u bytes\n",
r600_resource_va(&rscreen->b, &res->b.b),
r600_resource_va(&rscreen->b, &res->b.b) + res->buf->size,
res->buf->size);
}
return true;
}
static bool
r600_invalidate_buffer(struct r600_common_context *rctx,
struct r600_resource *rbuffer)
{
/* In AMD_pinned_memory, the user pointer association only gets
* broken when the buffer is explicitly re-allocated.
*/
if (rctx->ws->buffer_is_user_ptr(rbuffer->buf))
return false;
/* Check if mapping this buffer would cause waiting for the GPU. */
if (r600_rings_is_buffer_referenced(rctx, rbuffer->buf, RADEON_USAGE_READWRITE) ||
!rctx->ws->buffer_wait(rbuffer->buf, 0, RADEON_USAGE_READWRITE)) {
rctx->invalidate_buffer(&rctx->b, &rbuffer->b.b);
} else {
util_range_set_empty(&rbuffer->valid_buffer_range);
}
return true;
}
static void
realloc_bo(struct fd_resource *rsc, uint32_t size)
{
struct fd_screen *screen = fd_screen(rsc->base.b.screen);
uint32_t flags = DRM_FREEDRENO_GEM_CACHE_WCOMBINE |
DRM_FREEDRENO_GEM_TYPE_KMEM; /* TODO */
/* if we start using things other than write-combine,
* be sure to check for PIPE_RESOURCE_FLAG_MAP_COHERENT
*/
if (rsc->bo)
fd_bo_del(rsc->bo);
rsc->bo = fd_bo_new(screen->dev, size, flags);
rsc->timestamp = 0;
rsc->dirty = rsc->reading = false;
list_delinit(&rsc->list);
util_range_set_empty(&rsc->valid_buffer_range);
}
bool r600_alloc_resource(struct r600_common_screen *rscreen,
struct r600_resource *res)
{
struct pb_buffer *old_buf, *new_buf;
/* Allocate a new resource. */
new_buf = rscreen->ws->buffer_create(rscreen->ws, res->bo_size,
res->bo_alignment,
res->domains, res->flags);
if (!new_buf) {
return false;
}
/* Replace the pointer such that if res->buf wasn't NULL, it won't be
* NULL. This should prevent crashes with multiple contexts using
* the same buffer where one of the contexts invalidates it while
* the others are using it. */
old_buf = res->buf;
res->buf = new_buf; /* should be atomic */
if (rscreen->info.has_virtual_memory)
res->gpu_address = rscreen->ws->buffer_get_virtual_address(res->buf);
else
res->gpu_address = 0;
pb_reference(&old_buf, NULL);
util_range_set_empty(&res->valid_buffer_range);
res->TC_L2_dirty = false;
/* Print debug information. */
if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) {
fprintf(stderr, "VM start=0x%"PRIX64" end=0x%"PRIX64" | Buffer %"PRIu64" bytes\n",
res->gpu_address, res->gpu_address + res->buf->size,
res->buf->size);
}
return true;
}
static INLINE boolean
nouveau_buffer_allocate(struct nouveau_screen *screen,
struct nv04_resource *buf, unsigned domain)
{
uint32_t size = buf->base.width0;
if (buf->base.bind & (PIPE_BIND_CONSTANT_BUFFER |
PIPE_BIND_COMPUTE_RESOURCE |
PIPE_BIND_SHADER_RESOURCE))
size = align(size, 0x100);
if (domain == NOUVEAU_BO_VRAM) {
buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
&buf->bo, &buf->offset);
if (!buf->bo)
return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_vid, buf->base.width0);
} else
if (domain == NOUVEAU_BO_GART) {
buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
&buf->bo, &buf->offset);
if (!buf->bo)
return FALSE;
NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_sys, buf->base.width0);
} else {
assert(domain == 0);
if (!nouveau_buffer_malloc(buf))
return FALSE;
}
buf->domain = domain;
if (buf->bo)
buf->address = buf->bo->offset + buf->offset;
util_range_set_empty(&buf->valid_buffer_range);
return TRUE;
}
bool r600_init_resource(struct r600_common_screen *rscreen,
struct r600_resource *res,
unsigned size, unsigned alignment,
bool use_reusable_pool)
{
struct r600_texture *rtex = (struct r600_texture*)res;
struct pb_buffer *old_buf, *new_buf;
enum radeon_bo_flag flags = 0;
switch (res->b.b.usage) {
case PIPE_USAGE_STREAM:
flags = RADEON_FLAG_GTT_WC;
/* fall through */
case PIPE_USAGE_STAGING:
/* Transfers are likely to occur more often with these resources. */
res->domains = RADEON_DOMAIN_GTT;
break;
case PIPE_USAGE_DYNAMIC:
/* Older kernels didn't always flush the HDP cache before
* CS execution
*/
if (rscreen->info.drm_major == 2 &&
rscreen->info.drm_minor < 40) {
res->domains = RADEON_DOMAIN_GTT;
flags |= RADEON_FLAG_GTT_WC;
break;
}
flags |= RADEON_FLAG_CPU_ACCESS;
/* fall through */
case PIPE_USAGE_DEFAULT:
case PIPE_USAGE_IMMUTABLE:
default:
/* Not listing GTT here improves performance in some apps. */
res->domains = RADEON_DOMAIN_VRAM;
flags |= RADEON_FLAG_GTT_WC;
break;
}
if (res->b.b.target == PIPE_BUFFER &&
res->b.b.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
PIPE_RESOURCE_FLAG_MAP_COHERENT)) {
/* Use GTT for all persistent mappings with older kernels,
* because they didn't always flush the HDP cache before CS
* execution.
*
* Write-combined CPU mappings are fine, the kernel ensures all CPU
* writes finish before the GPU executes a command stream.
*/
if (rscreen->info.drm_major == 2 &&
rscreen->info.drm_minor < 40)
res->domains = RADEON_DOMAIN_GTT;
else if (res->domains & RADEON_DOMAIN_VRAM)
flags |= RADEON_FLAG_CPU_ACCESS;
}
/* Tiled textures are unmappable. Always put them in VRAM. */
if (res->b.b.target != PIPE_BUFFER &&
rtex->surface.level[0].mode >= RADEON_SURF_MODE_1D) {
res->domains = RADEON_DOMAIN_VRAM;
flags &= ~RADEON_FLAG_CPU_ACCESS;
flags |= RADEON_FLAG_NO_CPU_ACCESS;
}
if (rscreen->debug_flags & DBG_NO_WC)
flags &= ~RADEON_FLAG_GTT_WC;
/* Allocate a new resource. */
new_buf = rscreen->ws->buffer_create(rscreen->ws, size, alignment,
use_reusable_pool,
res->domains, flags);
if (!new_buf) {
return false;
}
/* Replace the pointer such that if res->buf wasn't NULL, it won't be
* NULL. This should prevent crashes with multiple contexts using
* the same buffer where one of the contexts invalidates it while
* the others are using it. */
old_buf = res->buf;
res->buf = new_buf; /* should be atomic */
if (rscreen->info.has_virtual_memory)
res->gpu_address = rscreen->ws->buffer_get_virtual_address(res->buf);
else
res->gpu_address = 0;
pb_reference(&old_buf, NULL);
util_range_set_empty(&res->valid_buffer_range);
res->TC_L2_dirty = false;
if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) {
fprintf(stderr, "VM start=0x%"PRIX64" end=0x%"PRIX64" | Buffer %u bytes\n",
res->gpu_address, res->gpu_address + res->buf->size,
res->buf->size);
}
return true;
}
bool r600_init_resource(struct r600_common_screen *rscreen,
struct r600_resource *res,
unsigned size, unsigned alignment,
bool use_reusable_pool)
{
struct r600_texture *rtex = (struct r600_texture*)res;
struct pb_buffer *old_buf, *new_buf;
switch (res->b.b.usage) {
case PIPE_USAGE_STAGING:
case PIPE_USAGE_DYNAMIC:
case PIPE_USAGE_STREAM:
/* Transfers are likely to occur more often with these resources. */
res->domains = RADEON_DOMAIN_GTT;
break;
case PIPE_USAGE_DEFAULT:
case PIPE_USAGE_IMMUTABLE:
default:
/* Not listing GTT here improves performance in some apps. */
res->domains = RADEON_DOMAIN_VRAM;
break;
}
/* Use GTT for all persistent mappings, because they are
* always cached and coherent. */
if (res->b.b.target == PIPE_BUFFER &&
res->b.b.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
PIPE_RESOURCE_FLAG_MAP_COHERENT)) {
res->domains = RADEON_DOMAIN_GTT;
}
/* Tiled textures are unmappable. Always put them in VRAM. */
if (res->b.b.target != PIPE_BUFFER &&
rtex->surface.level[0].mode >= RADEON_SURF_MODE_1D) {
res->domains = RADEON_DOMAIN_VRAM;
}
/* Allocate a new resource. */
new_buf = rscreen->ws->buffer_create(rscreen->ws, size, alignment,
use_reusable_pool,
res->domains);
if (!new_buf) {
return false;
}
/* Replace the pointer such that if res->buf wasn't NULL, it won't be
* NULL. This should prevent crashes with multiple contexts using
* the same buffer where one of the contexts invalidates it while
* the others are using it. */
old_buf = res->buf;
res->cs_buf = rscreen->ws->buffer_get_cs_handle(new_buf); /* should be atomic */
res->buf = new_buf; /* should be atomic */
pb_reference(&old_buf, NULL);
util_range_set_empty(&res->valid_buffer_range);
if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) {
fprintf(stderr, "VM start=0x%"PRIu64" end=0x%"PRIu64" | Buffer %u bytes\n",
r600_resource_va(&rscreen->b, &res->b.b),
r600_resource_va(&rscreen->b, &res->b.b) + res->buf->size,
res->buf->size);
}
return true;
}
