static void
radv_descriptor_set_destroy(struct radv_device *device,
struct radv_descriptor_pool *pool,
struct radv_descriptor_set *set,
bool free_bo)
{
if (free_bo && set->size) {
assert(pool->full_list >= 0);
int next = pool->free_nodes[pool->full_list].next;
pool->free_nodes[pool->full_list].next = pool->free_list;
pool->free_nodes[pool->full_list].offset = (uint8_t*)set->mapped_ptr - pool->mapped_ptr;
pool->free_nodes[pool->full_list].size = align_u32(set->size, 32);
pool->free_list = pool->full_list;
pool->full_list = next;
}
if (set->dynamic_descriptors)
vk_free2(&device->alloc, NULL, set->dynamic_descriptors);
if (!list_empty(&set->descriptor_pool))
list_del(&set->descriptor_pool);
vk_free2(&device->alloc, NULL, set);
}
void radv_DestroyPipelineLayout(
VkDevice _device,
VkPipelineLayout _pipelineLayout,
const VkAllocationCallbacks* pAllocator)
{
RADV_FROM_HANDLE(radv_device, device, _device);
RADV_FROM_HANDLE(radv_pipeline_layout, pipeline_layout, _pipelineLayout);
if (!pipeline_layout)
return;
vk_free2(&device->alloc, pAllocator, pipeline_layout);
}
void anv_DestroyQueryPool(
VkDevice _device,
VkQueryPool _pool,
const VkAllocationCallbacks* pAllocator)
{
ANV_FROM_HANDLE(anv_device, device, _device);
ANV_FROM_HANDLE(anv_query_pool, pool, _pool);
anv_gem_munmap(pool->bo.map, pool->bo.size);
anv_gem_close(device, pool->bo.gem_handle);
vk_free2(&device->alloc, pAllocator, pool);
}
void radv_DestroyDescriptorSetLayout(
VkDevice _device,
VkDescriptorSetLayout _set_layout,
const VkAllocationCallbacks* pAllocator)
{
RADV_FROM_HANDLE(radv_device, device, _device);
RADV_FROM_HANDLE(radv_descriptor_set_layout, set_layout, _set_layout);
if (!set_layout)
return;
vk_free2(&device->alloc, pAllocator, set_layout);
}
void
tu_DestroyPipelineCache(VkDevice _device,
VkPipelineCache _cache,
const VkAllocationCallbacks *pAllocator)
{
TU_FROM_HANDLE(tu_device, device, _device);
TU_FROM_HANDLE(tu_pipeline_cache, cache, _cache);
if (!cache)
return;
tu_pipeline_cache_finish(cache);
vk_free2(&device->alloc, pAllocator, cache);
}
VkResult anv_CreateQueryPool(
VkDevice _device,
const VkQueryPoolCreateInfo* pCreateInfo,
const VkAllocationCallbacks* pAllocator,
VkQueryPool* pQueryPool)
{
ANV_FROM_HANDLE(anv_device, device, _device);
struct anv_query_pool *pool;
VkResult result;
uint32_t slot_size;
uint64_t size;
assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO);
switch (pCreateInfo->queryType) {
case VK_QUERY_TYPE_OCCLUSION:
case VK_QUERY_TYPE_TIMESTAMP:
break;
case VK_QUERY_TYPE_PIPELINE_STATISTICS:
return VK_ERROR_INCOMPATIBLE_DRIVER;
default:
assert(!"Invalid query type");
}
slot_size = sizeof(struct anv_query_pool_slot);
pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (pool == NULL)
return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
pool->type = pCreateInfo->queryType;
pool->slots = pCreateInfo->queryCount;
size = pCreateInfo->queryCount * slot_size;
result = anv_bo_init_new(&pool->bo, device, size);
if (result != VK_SUCCESS)
goto fail;
pool->bo.map = anv_gem_mmap(device, pool->bo.gem_handle, 0, size, 0);
*pQueryPool = anv_query_pool_to_handle(pool);
return VK_SUCCESS;
fail:
vk_free2(&device->alloc, pAllocator, pool);
return result;
}
void radv_DestroyDescriptorPool(
VkDevice _device,
VkDescriptorPool _pool,
const VkAllocationCallbacks* pAllocator)
{
RADV_FROM_HANDLE(radv_device, device, _device);
RADV_FROM_HANDLE(radv_descriptor_pool, pool, _pool);
if (!pool)
return;
list_for_each_entry_safe(struct radv_descriptor_set, set,
&pool->descriptor_sets, descriptor_pool) {
radv_descriptor_set_destroy(device, pool, set, false);
}
if (pool->bo)
device->ws->buffer_destroy(pool->bo);
vk_free2(&device->alloc, pAllocator, pool);
}
void radv_DestroyDescriptorPool(
VkDevice _device,
VkDescriptorPool _pool,
const VkAllocationCallbacks* pAllocator)
{
RADV_FROM_HANDLE(radv_device, device, _device);
RADV_FROM_HANDLE(radv_descriptor_pool, pool, _pool);
if (!pool)
return;
if (!pool->host_memory_base) {
for(int i = 0; i < pool->entry_count; ++i) {
radv_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
}
}
if (pool->bo)
device->ws->buffer_destroy(pool->bo);
vk_free2(&device->alloc, pAllocator, pool);
}
static void
radv_descriptor_set_destroy(struct radv_device *device,
struct radv_descriptor_pool *pool,
struct radv_descriptor_set *set,
bool free_bo)
{
assert(!pool->host_memory_base);
if (free_bo && set->size && !pool->host_memory_base) {
uint32_t offset = (uint8_t*)set->mapped_ptr - pool->mapped_ptr;
for (int i = 0; i < pool->entry_count; ++i) {
if (pool->entries[i].offset == offset) {
memmove(&pool->entries[i], &pool->entries[i+1],
sizeof(pool->entries[i]) * (pool->entry_count - i - 1));
--pool->entry_count;
break;
}
}
}
vk_free2(&device->alloc, NULL, set);
}
static VkResult
compute_pipeline_create(
VkDevice _device,
struct anv_pipeline_cache * cache,
const VkComputePipelineCreateInfo* pCreateInfo,
const VkAllocationCallbacks* pAllocator,
VkPipeline* pPipeline)
{
ANV_FROM_HANDLE(anv_device, device, _device);
const struct anv_physical_device *physical_device =
&device->instance->physicalDevice;
const struct gen_device_info *devinfo = &physical_device->info;
struct anv_pipeline *pipeline;
VkResult result;
assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO);
pipeline = vk_alloc2(&device->alloc, pAllocator, sizeof(*pipeline), 8,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (pipeline == NULL)
return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
pipeline->device = device;
pipeline->layout = anv_pipeline_layout_from_handle(pCreateInfo->layout);
pipeline->blend_state.map = NULL;
result = anv_reloc_list_init(&pipeline->batch_relocs,
pAllocator ? pAllocator : &device->alloc);
if (result != VK_SUCCESS) {
vk_free2(&device->alloc, pAllocator, pipeline);
return result;
}
pipeline->batch.next = pipeline->batch.start = pipeline->batch_data;
pipeline->batch.end = pipeline->batch.start + sizeof(pipeline->batch_data);
pipeline->batch.relocs = &pipeline->batch_relocs;
/* When we free the pipeline, we detect stages based on the NULL status
* of various prog_data pointers. Make them NULL by default.
*/
memset(pipeline->shaders, 0, sizeof(pipeline->shaders));
pipeline->vs_simd8 = NO_KERNEL;
pipeline->vs_vec4 = NO_KERNEL;
pipeline->gs_kernel = NO_KERNEL;
pipeline->active_stages = 0;
pipeline->needs_data_cache = false;
assert(pCreateInfo->stage.stage == VK_SHADER_STAGE_COMPUTE_BIT);
ANV_FROM_HANDLE(anv_shader_module, module, pCreateInfo->stage.module);
result = anv_pipeline_compile_cs(pipeline, cache, pCreateInfo, module,
pCreateInfo->stage.pName,
pCreateInfo->stage.pSpecializationInfo);
if (result != VK_SUCCESS) {
vk_free2(&device->alloc, pAllocator, pipeline);
return result;
}
const struct brw_cs_prog_data *cs_prog_data = get_cs_prog_data(pipeline);
anv_pipeline_setup_l3_config(pipeline, cs_prog_data->base.total_shared > 0);
uint32_t group_size = cs_prog_data->local_size[0] *
cs_prog_data->local_size[1] * cs_prog_data->local_size[2];
uint32_t remainder = group_size & (cs_prog_data->simd_size - 1);
if (remainder > 0)
pipeline->cs_right_mask = ~0u >> (32 - remainder);
else
VkResult radv_CreateDescriptorUpdateTemplateKHR(VkDevice _device,
const VkDescriptorUpdateTemplateCreateInfoKHR *pCreateInfo,
const VkAllocationCallbacks *pAllocator,
VkDescriptorUpdateTemplateKHR *pDescriptorUpdateTemplate)
{
RADV_FROM_HANDLE(radv_device, device, _device);
RADV_FROM_HANDLE(radv_descriptor_set_layout, set_layout, pCreateInfo->descriptorSetLayout);
const uint32_t entry_count = pCreateInfo->descriptorUpdateEntryCount;
const size_t size = sizeof(struct radv_descriptor_update_template) +
sizeof(struct radv_descriptor_update_template_entry) * entry_count;
struct radv_descriptor_update_template *templ;
uint32_t i;
templ = vk_alloc2(&device->alloc, pAllocator, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!templ)
return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
templ->entry_count = entry_count;
for (i = 0; i < entry_count; i++) {
const VkDescriptorUpdateTemplateEntryKHR *entry = &pCreateInfo->pDescriptorUpdateEntries[i];
const struct radv_descriptor_set_binding_layout *binding_layout =
set_layout->binding + entry->dstBinding;
const uint32_t buffer_offset = binding_layout->buffer_offset + entry->dstArrayElement;
const uint32_t *immutable_samplers = NULL;
uint32_t dst_offset;
uint32_t dst_stride;
/* dst_offset is an offset into dynamic_descriptors when the descriptor
is dynamic, and an offset into mapped_ptr otherwise */
switch (entry->descriptorType) {
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
assert(pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET_KHR);
dst_offset = binding_layout->dynamic_offset_offset + entry->dstArrayElement;
dst_stride = 0; /* Not used */
break;
default:
switch (entry->descriptorType) {
case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
case VK_DESCRIPTOR_TYPE_SAMPLER:
/* Immutable samplers are copied into push descriptors when they are pushed */
if (pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR &&
binding_layout->immutable_samplers_offset && !binding_layout->immutable_samplers_equal) {
immutable_samplers = radv_immutable_samplers(set_layout, binding_layout) + entry->dstArrayElement * 4;
}
break;
default:
break;
}
dst_offset = binding_layout->offset / 4 + binding_layout->size * entry->dstArrayElement / 4;
dst_stride = binding_layout->size / 4;
break;
}
templ->entry[i] = (struct radv_descriptor_update_template_entry) {
.descriptor_type = entry->descriptorType,
.descriptor_count = entry->descriptorCount,
.src_offset = entry->offset,
.src_stride = entry->stride,
.dst_offset = dst_offset,
.dst_stride = dst_stride,
.buffer_offset = buffer_offset,
.has_sampler = !binding_layout->immutable_samplers_offset,
.immutable_samplers = immutable_samplers
};
}
*pDescriptorUpdateTemplate = radv_descriptor_update_template_to_handle(templ);
return VK_SUCCESS;
}
void radv_DestroyDescriptorUpdateTemplateKHR(VkDevice _device,
VkDescriptorUpdateTemplateKHR descriptorUpdateTemplate,
const VkAllocationCallbacks *pAllocator)
{
RADV_FROM_HANDLE(radv_device, device, _device);
RADV_FROM_HANDLE(radv_descriptor_update_template, templ, descriptorUpdateTemplate);
if (!templ)
return;
vk_free2(&device->alloc, pAllocator, templ);
}
static VkResult
radv_descriptor_set_create(struct radv_device *device,
struct radv_descriptor_pool *pool,
const struct radv_descriptor_set_layout *layout,
struct radv_descriptor_set **out_set)
{
struct radv_descriptor_set *set;
unsigned range_offset = sizeof(struct radv_descriptor_set) +
sizeof(struct radeon_winsys_bo *) * layout->buffer_count;
unsigned mem_size = range_offset +
sizeof(struct radv_descriptor_range) * layout->dynamic_offset_count;
if (pool->host_memory_base) {
if (pool->host_memory_end - pool->host_memory_ptr < mem_size)
return vk_error(VK_ERROR_OUT_OF_POOL_MEMORY_KHR);
set = (struct radv_descriptor_set*)pool->host_memory_ptr;
pool->host_memory_ptr += mem_size;
} else {
set = vk_alloc2(&device->alloc, NULL, mem_size, 8,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!set)
return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
}
memset(set, 0, mem_size);
if (layout->dynamic_offset_count) {
set->dynamic_descriptors = (struct radv_descriptor_range*)((uint8_t*)set + range_offset);
}
set->layout = layout;
if (layout->size) {
uint32_t layout_size = align_u32(layout->size, 32);
set->size = layout->size;
if (!pool->host_memory_base && pool->entry_count == pool->max_entry_count) {
vk_free2(&device->alloc, NULL, set);
return vk_error(VK_ERROR_OUT_OF_POOL_MEMORY_KHR);
}
/* try to allocate linearly first, so that we don't spend
* time looking for gaps if the app only allocates &
* resets via the pool. */
if (pool->current_offset + layout_size <= pool->size) {
set->bo = pool->bo;
set->mapped_ptr = (uint32_t*)(pool->mapped_ptr + pool->current_offset);
set->va = radv_buffer_get_va(set->bo) + pool->current_offset;
if (!pool->host_memory_base) {
pool->entries[pool->entry_count].offset = pool->current_offset;
pool->entries[pool->entry_count].size = layout_size;
pool->entries[pool->entry_count].set = set;
pool->entry_count++;
}
pool->current_offset += layout_size;
} else if (!pool->host_memory_base) {
uint64_t offset = 0;
int index;
for (index = 0; index < pool->entry_count; ++index) {
if (pool->entries[index].offset - offset >= layout_size)
break;
offset = pool->entries[index].offset + pool->entries[index].size;
}
if (pool->size - offset < layout_size) {
vk_free2(&device->alloc, NULL, set);
return vk_error(VK_ERROR_OUT_OF_POOL_MEMORY_KHR);
}
set->bo = pool->bo;
set->mapped_ptr = (uint32_t*)(pool->mapped_ptr + offset);
set->va = radv_buffer_get_va(set->bo) + offset;
memmove(&pool->entries[index + 1], &pool->entries[index],
sizeof(pool->entries[0]) * (pool->entry_count - index));
pool->entries[index].offset = offset;
pool->entries[index].size = layout_size;
pool->entries[index].set = set;
pool->entry_count++;
} else
return vk_error(VK_ERROR_OUT_OF_POOL_MEMORY_KHR);
}
if (layout->has_immutable_samplers) {
for (unsigned i = 0; i < layout->binding_count; ++i) {
if (!layout->binding[i].immutable_samplers_offset ||
layout->binding[i].immutable_samplers_equal)
continue;
unsigned offset = layout->binding[i].offset / 4;
if (layout->binding[i].type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
offset += 16;
const uint32_t *samplers = (const uint32_t*)((const char*)layout + layout->binding[i].immutable_samplers_offset);
for (unsigned j = 0; j < layout->binding[i].array_size; ++j) {
memcpy(set->mapped_ptr + offset, samplers + 4 * j, 16);
offset += layout->binding[i].size / 4;
}
}
}
*out_set = set;
return VK_SUCCESS;
}
static VkResult
radv_descriptor_set_create(struct radv_device *device,
struct radv_descriptor_pool *pool,
struct radv_cmd_buffer *cmd_buffer,
const struct radv_descriptor_set_layout *layout,
struct radv_descriptor_set **out_set)
{
struct radv_descriptor_set *set;
unsigned mem_size = sizeof(struct radv_descriptor_set) +
sizeof(struct radeon_winsys_bo *) * layout->buffer_count;
set = vk_alloc2(&device->alloc, NULL, mem_size, 8,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!set)
return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
memset(set, 0, mem_size);
if (layout->dynamic_offset_count) {
unsigned size = sizeof(struct radv_descriptor_range) *
layout->dynamic_offset_count;
set->dynamic_descriptors = vk_alloc2(&device->alloc, NULL, size, 8,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!set->dynamic_descriptors) {
vk_free2(&device->alloc, NULL, set);
return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
}
}
set->layout = layout;
if (layout->size) {
uint32_t layout_size = align_u32(layout->size, 32);
set->size = layout->size;
if (!cmd_buffer) {
if (pool->current_offset + layout_size <= pool->size &&
pool->allocated_sets < pool->max_sets) {
set->bo = pool->bo;
set->mapped_ptr = (uint32_t*)(pool->mapped_ptr + pool->current_offset);
set->va = device->ws->buffer_get_va(set->bo) + pool->current_offset;
pool->current_offset += layout_size;
++pool->allocated_sets;
} else {
int entry = pool->free_list, prev_entry = -1;
uint32_t offset;
while (entry >= 0) {
if (pool->free_nodes[entry].size >= layout_size) {
if (prev_entry >= 0)
pool->free_nodes[prev_entry].next = pool->free_nodes[entry].next;
else
pool->free_list = pool->free_nodes[entry].next;
break;
}
prev_entry = entry;
entry = pool->free_nodes[entry].next;
}
if (entry < 0) {
vk_free2(&device->alloc, NULL, set);
return vk_error(VK_ERROR_OUT_OF_DEVICE_MEMORY);
}
offset = pool->free_nodes[entry].offset;
pool->free_nodes[entry].next = pool->full_list;
pool->full_list = entry;
set->bo = pool->bo;
set->mapped_ptr = (uint32_t*)(pool->mapped_ptr + offset);
set->va = device->ws->buffer_get_va(set->bo) + offset;
}
} else {
unsigned bo_offset;
if (!radv_cmd_buffer_upload_alloc(cmd_buffer, set->size, 32,
&bo_offset,
(void**)&set->mapped_ptr)) {
vk_free2(&device->alloc, NULL, set->dynamic_descriptors);
vk_free2(&device->alloc, NULL, set);
return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
}
set->va = device->ws->buffer_get_va(cmd_buffer->upload.upload_bo);
set->va += bo_offset;
}
}
if (pool)
list_add(&set->descriptor_pool, &pool->descriptor_sets);
else
list_inithead(&set->descriptor_pool);
for (unsigned i = 0; i < layout->binding_count; ++i) {
if (!layout->binding[i].immutable_samplers)
continue;
unsigned offset = layout->binding[i].offset / 4;
if (layout->binding[i].type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
offset += 16;
for (unsigned j = 0; j < layout->binding[i].array_size; ++j) {
struct radv_sampler* sampler = layout->binding[i].immutable_samplers[j];
memcpy(set->mapped_ptr + offset, &sampler->state, 16);
offset += layout->binding[i].size / 4;
}
}
*out_set = set;
return VK_SUCCESS;
}
