static inline bool nouveau_buffer_allocate(struct nouveau_screen *screen, struct nv04_resource *buf, unsigned domain) { uint32_t size = align(buf->base.width0, 0x100); if (domain == NOUVEAU_BO_VRAM) { buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size, &buf->bo, &buf->offset); if (!buf->bo) return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART); NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_vid, buf->base.width0); } else if (domain == NOUVEAU_BO_GART) { buf->mm = nouveau_mm_allocate(screen->mm_GART, size, &buf->bo, &buf->offset); if (!buf->bo) return false; NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_sys, buf->base.width0); } else { assert(domain == 0); if (!nouveau_buffer_malloc(buf)) return false; } buf->domain = domain; if (buf->bo) buf->address = buf->bo->offset + buf->offset; util_range_set_empty(&buf->valid_buffer_range); return true; }
/* Invalidate underlying buffer storage, reset fences, reallocate to non-busy * buffer. */ void nouveau_buffer_invalidate(struct pipe_context *pipe, struct pipe_resource *resource) { struct nouveau_context *nv = nouveau_context(pipe); struct nv04_resource *buf = nv04_resource(resource); int ref = buf->base.reference.count - 1; /* Shared buffers shouldn't get reallocated */ if (unlikely(buf->base.bind & PIPE_BIND_SHARED)) return; /* We can't touch persistent/coherent buffers */ if (buf->base.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT | PIPE_RESOURCE_FLAG_MAP_COHERENT)) return; /* If the buffer is sub-allocated and not currently being written, just * wipe the valid buffer range. Otherwise we have to create fresh * storage. (We don't keep track of fences for non-sub-allocated BO's.) */ if (buf->mm && !nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE)) { util_range_set_empty(&buf->valid_buffer_range); } else { nouveau_buffer_reallocate(nv->screen, buf, buf->domain); if (ref > 0) /* any references inside context possible ? */ nv->invalidate_resource_storage(nv, &buf->base, ref); } }
bool r600_init_resource(struct r600_common_screen *rscreen, struct r600_resource *res, unsigned size, unsigned alignment, bool use_reusable_pool, unsigned usage) { uint32_t initial_domain, domains; switch(usage) { case PIPE_USAGE_STAGING: /* Staging resources participate in transfers, i.e. are used * for uploads and downloads from regular resources. * We generate them internally for some transfers. */ initial_domain = RADEON_DOMAIN_GTT; domains = RADEON_DOMAIN_GTT; break; case PIPE_USAGE_DYNAMIC: case PIPE_USAGE_STREAM: /* Default to GTT, but allow the memory manager to move it to VRAM. */ initial_domain = RADEON_DOMAIN_GTT; domains = RADEON_DOMAIN_GTT | RADEON_DOMAIN_VRAM; break; case PIPE_USAGE_DEFAULT: case PIPE_USAGE_STATIC: case PIPE_USAGE_IMMUTABLE: default: /* Don't list GTT here, because the memory manager would put some * resources to GTT no matter what the initial domain is. * Not listing GTT in the domains improves performance a lot. */ initial_domain = RADEON_DOMAIN_VRAM; domains = RADEON_DOMAIN_VRAM; break; } res->buf = rscreen->ws->buffer_create(rscreen->ws, size, alignment, use_reusable_pool, initial_domain); if (!res->buf) { return false; } res->cs_buf = rscreen->ws->buffer_get_cs_handle(res->buf); res->domains = domains; util_range_set_empty(&res->valid_buffer_range); if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) { fprintf(stderr, "VM start=0x%"PRIu64" end=0x%"PRIu64" | Buffer %u bytes\n", r600_resource_va(&rscreen->b, &res->b.b), r600_resource_va(&rscreen->b, &res->b.b) + res->buf->size, res->buf->size); } return true; }
bool r600_init_resource(struct r600_common_screen *rscreen, struct r600_resource *res, unsigned size, unsigned alignment, bool use_reusable_pool) { struct r600_texture *rtex = (struct r600_texture*)res; switch (res->b.b.usage) { case PIPE_USAGE_STAGING: case PIPE_USAGE_DYNAMIC: case PIPE_USAGE_STREAM: /* Transfers are likely to occur more often with these resources. */ res->domains = RADEON_DOMAIN_GTT; break; case PIPE_USAGE_DEFAULT: case PIPE_USAGE_IMMUTABLE: default: /* Not listing GTT here improves performance in some apps. */ res->domains = RADEON_DOMAIN_VRAM; break; } /* Tiled textures are unmappable. Always put them in VRAM. */ if (res->b.b.target != PIPE_BUFFER && rtex->surface.level[0].mode >= RADEON_SURF_MODE_1D) { res->domains = RADEON_DOMAIN_VRAM; } /* Allocate the resource. */ res->buf = rscreen->ws->buffer_create(rscreen->ws, size, alignment, use_reusable_pool, res->domains); if (!res->buf) { return false; } res->cs_buf = rscreen->ws->buffer_get_cs_handle(res->buf); util_range_set_empty(&res->valid_buffer_range); if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) { fprintf(stderr, "VM start=0x%"PRIu64" end=0x%"PRIu64" | Buffer %u bytes\n", r600_resource_va(&rscreen->b, &res->b.b), r600_resource_va(&rscreen->b, &res->b.b) + res->buf->size, res->buf->size); } return true; }
static bool r600_invalidate_buffer(struct r600_common_context *rctx, struct r600_resource *rbuffer) { /* In AMD_pinned_memory, the user pointer association only gets * broken when the buffer is explicitly re-allocated. */ if (rctx->ws->buffer_is_user_ptr(rbuffer->buf)) return false; /* Check if mapping this buffer would cause waiting for the GPU. */ if (r600_rings_is_buffer_referenced(rctx, rbuffer->buf, RADEON_USAGE_READWRITE) || !rctx->ws->buffer_wait(rbuffer->buf, 0, RADEON_USAGE_READWRITE)) { rctx->invalidate_buffer(&rctx->b, &rbuffer->b.b); } else { util_range_set_empty(&rbuffer->valid_buffer_range); } return true; }
static void realloc_bo(struct fd_resource *rsc, uint32_t size) { struct fd_screen *screen = fd_screen(rsc->base.b.screen); uint32_t flags = DRM_FREEDRENO_GEM_CACHE_WCOMBINE | DRM_FREEDRENO_GEM_TYPE_KMEM; /* TODO */ /* if we start using things other than write-combine, * be sure to check for PIPE_RESOURCE_FLAG_MAP_COHERENT */ if (rsc->bo) fd_bo_del(rsc->bo); rsc->bo = fd_bo_new(screen->dev, size, flags); rsc->timestamp = 0; rsc->dirty = rsc->reading = false; list_delinit(&rsc->list); util_range_set_empty(&rsc->valid_buffer_range); }
bool r600_alloc_resource(struct r600_common_screen *rscreen, struct r600_resource *res) { struct pb_buffer *old_buf, *new_buf; /* Allocate a new resource. */ new_buf = rscreen->ws->buffer_create(rscreen->ws, res->bo_size, res->bo_alignment, res->domains, res->flags); if (!new_buf) { return false; } /* Replace the pointer such that if res->buf wasn't NULL, it won't be * NULL. This should prevent crashes with multiple contexts using * the same buffer where one of the contexts invalidates it while * the others are using it. */ old_buf = res->buf; res->buf = new_buf; /* should be atomic */ if (rscreen->info.has_virtual_memory) res->gpu_address = rscreen->ws->buffer_get_virtual_address(res->buf); else res->gpu_address = 0; pb_reference(&old_buf, NULL); util_range_set_empty(&res->valid_buffer_range); res->TC_L2_dirty = false; /* Print debug information. */ if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) { fprintf(stderr, "VM start=0x%"PRIX64" end=0x%"PRIX64" | Buffer %"PRIu64" bytes\n", res->gpu_address, res->gpu_address + res->buf->size, res->buf->size); } return true; }
static INLINE boolean nouveau_buffer_allocate(struct nouveau_screen *screen, struct nv04_resource *buf, unsigned domain) { uint32_t size = buf->base.width0; if (buf->base.bind & (PIPE_BIND_CONSTANT_BUFFER | PIPE_BIND_COMPUTE_RESOURCE | PIPE_BIND_SHADER_RESOURCE)) size = align(size, 0x100); if (domain == NOUVEAU_BO_VRAM) { buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size, &buf->bo, &buf->offset); if (!buf->bo) return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART); NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_vid, buf->base.width0); } else if (domain == NOUVEAU_BO_GART) { buf->mm = nouveau_mm_allocate(screen->mm_GART, size, &buf->bo, &buf->offset); if (!buf->bo) return FALSE; NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_sys, buf->base.width0); } else { assert(domain == 0); if (!nouveau_buffer_malloc(buf)) return FALSE; } buf->domain = domain; if (buf->bo) buf->address = buf->bo->offset + buf->offset; util_range_set_empty(&buf->valid_buffer_range); return TRUE; }
bool r600_init_resource(struct r600_common_screen *rscreen, struct r600_resource *res, unsigned size, unsigned alignment, bool use_reusable_pool) { struct r600_texture *rtex = (struct r600_texture*)res; struct pb_buffer *old_buf, *new_buf; enum radeon_bo_flag flags = 0; switch (res->b.b.usage) { case PIPE_USAGE_STREAM: flags = RADEON_FLAG_GTT_WC; /* fall through */ case PIPE_USAGE_STAGING: /* Transfers are likely to occur more often with these resources. */ res->domains = RADEON_DOMAIN_GTT; break; case PIPE_USAGE_DYNAMIC: /* Older kernels didn't always flush the HDP cache before * CS execution */ if (rscreen->info.drm_major == 2 && rscreen->info.drm_minor < 40) { res->domains = RADEON_DOMAIN_GTT; flags |= RADEON_FLAG_GTT_WC; break; } flags |= RADEON_FLAG_CPU_ACCESS; /* fall through */ case PIPE_USAGE_DEFAULT: case PIPE_USAGE_IMMUTABLE: default: /* Not listing GTT here improves performance in some apps. */ res->domains = RADEON_DOMAIN_VRAM; flags |= RADEON_FLAG_GTT_WC; break; } if (res->b.b.target == PIPE_BUFFER && res->b.b.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT | PIPE_RESOURCE_FLAG_MAP_COHERENT)) { /* Use GTT for all persistent mappings with older kernels, * because they didn't always flush the HDP cache before CS * execution. * * Write-combined CPU mappings are fine, the kernel ensures all CPU * writes finish before the GPU executes a command stream. */ if (rscreen->info.drm_major == 2 && rscreen->info.drm_minor < 40) res->domains = RADEON_DOMAIN_GTT; else if (res->domains & RADEON_DOMAIN_VRAM) flags |= RADEON_FLAG_CPU_ACCESS; } /* Tiled textures are unmappable. Always put them in VRAM. */ if (res->b.b.target != PIPE_BUFFER && rtex->surface.level[0].mode >= RADEON_SURF_MODE_1D) { res->domains = RADEON_DOMAIN_VRAM; flags &= ~RADEON_FLAG_CPU_ACCESS; flags |= RADEON_FLAG_NO_CPU_ACCESS; } if (rscreen->debug_flags & DBG_NO_WC) flags &= ~RADEON_FLAG_GTT_WC; /* Allocate a new resource. */ new_buf = rscreen->ws->buffer_create(rscreen->ws, size, alignment, use_reusable_pool, res->domains, flags); if (!new_buf) { return false; } /* Replace the pointer such that if res->buf wasn't NULL, it won't be * NULL. This should prevent crashes with multiple contexts using * the same buffer where one of the contexts invalidates it while * the others are using it. */ old_buf = res->buf; res->buf = new_buf; /* should be atomic */ if (rscreen->info.has_virtual_memory) res->gpu_address = rscreen->ws->buffer_get_virtual_address(res->buf); else res->gpu_address = 0; pb_reference(&old_buf, NULL); util_range_set_empty(&res->valid_buffer_range); res->TC_L2_dirty = false; if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) { fprintf(stderr, "VM start=0x%"PRIX64" end=0x%"PRIX64" | Buffer %u bytes\n", res->gpu_address, res->gpu_address + res->buf->size, res->buf->size); } return true; }
bool r600_init_resource(struct r600_common_screen *rscreen, struct r600_resource *res, unsigned size, unsigned alignment, bool use_reusable_pool) { struct r600_texture *rtex = (struct r600_texture*)res; struct pb_buffer *old_buf, *new_buf; switch (res->b.b.usage) { case PIPE_USAGE_STAGING: case PIPE_USAGE_DYNAMIC: case PIPE_USAGE_STREAM: /* Transfers are likely to occur more often with these resources. */ res->domains = RADEON_DOMAIN_GTT; break; case PIPE_USAGE_DEFAULT: case PIPE_USAGE_IMMUTABLE: default: /* Not listing GTT here improves performance in some apps. */ res->domains = RADEON_DOMAIN_VRAM; break; } /* Use GTT for all persistent mappings, because they are * always cached and coherent. */ if (res->b.b.target == PIPE_BUFFER && res->b.b.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT | PIPE_RESOURCE_FLAG_MAP_COHERENT)) { res->domains = RADEON_DOMAIN_GTT; } /* Tiled textures are unmappable. Always put them in VRAM. */ if (res->b.b.target != PIPE_BUFFER && rtex->surface.level[0].mode >= RADEON_SURF_MODE_1D) { res->domains = RADEON_DOMAIN_VRAM; } /* Allocate a new resource. */ new_buf = rscreen->ws->buffer_create(rscreen->ws, size, alignment, use_reusable_pool, res->domains); if (!new_buf) { return false; } /* Replace the pointer such that if res->buf wasn't NULL, it won't be * NULL. This should prevent crashes with multiple contexts using * the same buffer where one of the contexts invalidates it while * the others are using it. */ old_buf = res->buf; res->cs_buf = rscreen->ws->buffer_get_cs_handle(new_buf); /* should be atomic */ res->buf = new_buf; /* should be atomic */ pb_reference(&old_buf, NULL); util_range_set_empty(&res->valid_buffer_range); if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) { fprintf(stderr, "VM start=0x%"PRIu64" end=0x%"PRIu64" | Buffer %u bytes\n", r600_resource_va(&rscreen->b, &res->b.b), r600_resource_va(&rscreen->b, &res->b.b) + res->buf->size, res->buf->size); } return true; }