static int
vc4_simulator_pin_bos(struct drm_device *dev, struct vc4_exec_info *exec)
{
struct drm_vc4_submit_cl *args = exec->args;
struct vc4_context *vc4 = dev->vc4;
struct vc4_bo **bos = vc4->bo_pointers.base;
exec->bo_count = args->bo_handle_count;
exec->bo = calloc(exec->bo_count, sizeof(void *));
for (int i = 0; i < exec->bo_count; i++) {
struct vc4_bo *bo = bos[i];
struct drm_gem_cma_object *obj = vc4_wrap_bo_with_cma(dev, bo);
struct drm_vc4_bo *drm_bo = to_vc4_bo(&obj->base);
#if 0
fprintf(stderr, "bo hindex %d: %s\n", i, bo->name);
#endif
vc4_bo_map(bo);
memcpy(obj->vaddr, bo->map, bo->size);
exec->bo[i] = obj;
/* The kernel does this validation at shader create ioctl
* time.
*/
if (strcmp(bo->name, "code") == 0) {
drm_bo->validated_shader = vc4_validate_shader(obj);
if (!drm_bo->validated_shader)
abort();
}
}
return 0;
}
int vc4_mmap(struct file *filp, struct vm_area_struct *vma)
{
struct drm_gem_object *gem_obj;
struct vc4_bo *bo;
int ret;
ret = drm_gem_mmap(filp, vma);
if (ret)
return ret;
gem_obj = vma->vm_private_data;
bo = to_vc4_bo(gem_obj);
if (bo->validated_shader && (vma->vm_flags & VM_WRITE)) {
DRM_ERROR("mmaping of shader BOs for writing not allowed.\n");
return -EINVAL;
}
/*
* Clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the
* vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map
* the whole buffer.
*/
vma->vm_flags &= ~VM_PFNMAP;
vma->vm_pgoff = 0;
ret = dma_mmap_writecombine(bo->base.base.dev->dev, vma,
bo->base.vaddr, bo->base.paddr,
vma->vm_end - vma->vm_start);
if (ret)
drm_gem_vm_close(vma);
return ret;
}
void *vc4_prime_vmap(struct drm_gem_object *obj)
{
struct vc4_bo *bo = to_vc4_bo(obj);
if (bo->validated_shader) {
DRM_ERROR("mmaping of shader BOs not allowed.\n");
return ERR_PTR(-EINVAL);
}
return drm_gem_cma_prime_vmap(obj);
}
int vc4_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
{
struct vc4_bo *bo = to_vc4_bo(obj);
if (bo->validated_shader && (vma->vm_flags & VM_WRITE)) {
DRM_ERROR("mmaping of shader BOs for writing not allowed.\n");
return -EINVAL;
}
return drm_gem_cma_prime_mmap(obj, vma);
}
struct dma_buf *
vc4_prime_export(struct drm_device *dev, struct drm_gem_object *obj, int flags)
{
struct vc4_bo *bo = to_vc4_bo(obj);
if (bo->validated_shader) {
DRM_ERROR("Attempting to export shader BO\n");
return ERR_PTR(-EINVAL);
}
return drm_gem_prime_export(dev, obj, flags);
}
/* Called on the last userspace/kernel unreference of the BO. Returns
* it to the BO cache if possible, otherwise frees it.
*/
void vc4_free_object(struct drm_gem_object *gem_bo)
{
struct drm_device *dev = gem_bo->dev;
struct vc4_dev *vc4 = to_vc4_dev(dev);
struct vc4_bo *bo = to_vc4_bo(gem_bo);
struct list_head *cache_list;
mutex_lock(&vc4->bo_lock);
/* If the object references someone else's memory, we can't cache it.
*/
if (gem_bo->import_attach) {
vc4_bo_destroy(bo);
goto out;
}
/* Don't cache if it was publicly named. */
if (gem_bo->name) {
vc4_bo_destroy(bo);
goto out;
}
/* If this object was partially constructed but CMA allocation
* had failed, just free it.
*/
if (!bo->base.vaddr) {
vc4_bo_destroy(bo);
goto out;
}
cache_list = vc4_get_cache_list_for_size(dev, gem_bo->size);
if (!cache_list) {
vc4_bo_destroy(bo);
goto out;
}
if (bo->validated_shader) {
kfree(bo->validated_shader->texture_samples);
kfree(bo->validated_shader);
bo->validated_shader = NULL;
}
bo->free_time = jiffies;
list_add(&bo->size_head, cache_list);
list_add(&bo->unref_head, &vc4->bo_cache.time_list);
vc4->bo_stats.num_cached++;
vc4->bo_stats.size_cached += gem_bo->size;
vc4_bo_cache_free_old(dev);
out:
mutex_unlock(&vc4->bo_lock);
}
static int
vc4_simulator_unpin_bos(struct vc4_exec_info *exec)
{
for (int i = 0; i < exec->bo_count; i++) {
struct drm_gem_cma_object *obj = exec->bo[i];
struct drm_vc4_bo *drm_bo = to_vc4_bo(&obj->base);
struct vc4_bo *bo = drm_bo->bo;
assert(*(uint32_t *)(obj->vaddr + bo->size) == BO_SENTINEL);
memcpy(bo->map, obj->vaddr, bo->size);
if (drm_bo->validated_shader) {
free(drm_bo->validated_shader->texture_samples);
free(drm_bo->validated_shader);
}
free(obj);
}
free(exec->bo);
return 0;
}
struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t unaligned_size,
bool allow_unzeroed)
{
size_t size = roundup(unaligned_size, PAGE_SIZE);
struct vc4_dev *vc4 = to_vc4_dev(dev);
struct drm_gem_cma_object *cma_obj;
struct vc4_bo *bo;
if (size == 0)
return ERR_PTR(-EINVAL);
/* First, try to get a vc4_bo from the kernel BO cache. */
bo = vc4_bo_get_from_cache(dev, size);
if (bo) {
if (!allow_unzeroed)
memset(bo->base.vaddr, 0, bo->base.base.size);
return bo;
}
cma_obj = drm_gem_cma_create(dev, size);
if (IS_ERR(cma_obj)) {
/*
* If we've run out of CMA memory, kill the cache of
* CMA allocations we've got laying around and try again.
*/
vc4_bo_cache_purge(dev);
cma_obj = drm_gem_cma_create(dev, size);
if (IS_ERR(cma_obj)) {
DRM_ERROR("Failed to allocate from CMA:\n");
vc4_bo_stats_dump(vc4);
return ERR_PTR(-ENOMEM);
}
}
return to_vc4_bo(&cma_obj->base);
}
static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec,
struct vc4_rcl_setup *setup)
{
struct drm_vc4_submit_cl *args = exec->args;
bool has_bin = args->bin_cl_size != 0;
uint8_t min_x_tile = args->min_x_tile;
uint8_t min_y_tile = args->min_y_tile;
uint8_t max_x_tile = args->max_x_tile;
uint8_t max_y_tile = args->max_y_tile;
uint8_t xtiles = max_x_tile - min_x_tile + 1;
uint8_t ytiles = max_y_tile - min_y_tile + 1;
uint8_t x, y;
uint32_t size, loop_body_size;
size = VC4_PACKET_TILE_RENDERING_MODE_CONFIG_SIZE;
loop_body_size = VC4_PACKET_TILE_COORDINATES_SIZE;
if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) {
size += VC4_PACKET_CLEAR_COLORS_SIZE +
VC4_PACKET_TILE_COORDINATES_SIZE +
VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
}
if (setup->color_read) {
if (args->color_read.flags &
VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE;
} else {
loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE;
}
}
if (setup->zs_read) {
if (args->zs_read.flags &
VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE;
} else {
if (setup->color_read &&
!(args->color_read.flags &
VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES)) {
loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE;
loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
}
loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE;
}
}
if (has_bin) {
size += VC4_PACKET_WAIT_ON_SEMAPHORE_SIZE;
loop_body_size += VC4_PACKET_BRANCH_TO_SUB_LIST_SIZE;
}
if (setup->msaa_color_write)
loop_body_size += VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE;
if (setup->msaa_zs_write)
loop_body_size += VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE;
if (setup->zs_write)
loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
if (setup->color_write)
loop_body_size += VC4_PACKET_STORE_MS_TILE_BUFFER_SIZE;
/* We need a VC4_PACKET_TILE_COORDINATES in between each store. */
loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE *
((setup->msaa_color_write != NULL) +
(setup->msaa_zs_write != NULL) +
(setup->color_write != NULL) +
(setup->zs_write != NULL) - 1);
size += xtiles * ytiles * loop_body_size;
setup->rcl = drm_gem_cma_create(dev, size);
if (!setup->rcl)
return -ENOMEM;
list_addtail(&to_vc4_bo(&setup->rcl->base)->unref_head,
&exec->unref_list);
rcl_u8(setup, VC4_PACKET_TILE_RENDERING_MODE_CONFIG);
rcl_u32(setup,
(setup->color_write ? (setup->color_write->paddr +
args->color_write.offset) :
0));
rcl_u16(setup, args->width);
rcl_u16(setup, args->height);
rcl_u16(setup, args->color_write.bits);
/* The tile buffer gets cleared when the previous tile is stored. If
* the clear values changed between frames, then the tile buffer has
* stale clear values in it, so we have to do a store in None mode (no
* writes) so that we trigger the tile buffer clear.
*/
if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) {
rcl_u8(setup, VC4_PACKET_CLEAR_COLORS);
rcl_u32(setup, args->clear_color[0]);
rcl_u32(setup, args->clear_color[1]);
rcl_u32(setup, args->clear_z);
rcl_u8(setup, args->clear_s);
vc4_tile_coordinates(setup, 0, 0);
rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
rcl_u16(setup, VC4_LOADSTORE_TILE_BUFFER_NONE);
rcl_u32(setup, 0); /* no address, since we're in None mode */
}
for (y = min_y_tile; y <= max_y_tile; y++) {
for (x = min_x_tile; x <= max_x_tile; x++) {
bool first = (x == min_x_tile && y == min_y_tile);
bool last = (x == max_x_tile && y == max_y_tile);
emit_tile(exec, setup, x, y, first, last);
}
}
BUG_ON(setup->next_offset != size);
exec->ct1ca = setup->rcl->paddr;
exec->ct1ea = setup->rcl->paddr + setup->next_offset;
return 0;
}
/*
* Copies in the user's binning command list and generates the validated bin
* CL, along with associated data (shader records, uniforms).
*/
static int
vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec)
{
struct drm_vc4_submit_cl *args = exec->args;
void *temp = NULL;
void *bin;
int ret = 0;
uint32_t bin_offset = 0;
uint32_t shader_rec_offset = roundup(bin_offset + args->bin_cl_size,
16);
uint32_t uniforms_offset = shader_rec_offset + args->shader_rec_size;
uint32_t exec_size = uniforms_offset + args->uniforms_size;
uint32_t temp_size = exec_size + (sizeof(struct vc4_shader_state) *
args->shader_rec_count);
if (uniforms_offset < shader_rec_offset ||
exec_size < uniforms_offset ||
args->shader_rec_count >= (UINT_MAX /
sizeof(struct vc4_shader_state)) ||
temp_size < exec_size) {
DRM_ERROR("overflow in exec arguments\n");
goto fail;
}
/* Allocate space where we'll store the copied in user command lists
* and shader records.
*
* We don't just copy directly into the BOs because we need to
* read the contents back for validation, and I think the
* bo->vaddr is uncached access.
*/
temp = kmalloc(temp_size, GFP_KERNEL);
if (!temp) {
DRM_ERROR("Failed to allocate storage for copying "
"in bin/render CLs.\n");
ret = -ENOMEM;
goto fail;
}
bin = temp + bin_offset;
exec->shader_rec_u = temp + shader_rec_offset;
exec->uniforms_u = temp + uniforms_offset;
exec->shader_state = temp + exec_size;
exec->shader_state_size = args->shader_rec_count;
ret = copy_from_user(bin,
(void __user *)(uintptr_t)args->bin_cl,
args->bin_cl_size);
if (ret) {
DRM_ERROR("Failed to copy in bin cl\n");
goto fail;
}
ret = copy_from_user(exec->shader_rec_u,
(void __user *)(uintptr_t)args->shader_rec,
args->shader_rec_size);
if (ret) {
DRM_ERROR("Failed to copy in shader recs\n");
goto fail;
}
ret = copy_from_user(exec->uniforms_u,
(void __user *)(uintptr_t)args->uniforms,
args->uniforms_size);
if (ret) {
DRM_ERROR("Failed to copy in uniforms cl\n");
goto fail;
}
exec->exec_bo = drm_gem_cma_create(dev, exec_size);
#if 0
if (IS_ERR(exec->exec_bo)) {
DRM_ERROR("Couldn't allocate BO for exec\n");
ret = PTR_ERR(exec->exec_bo);
exec->exec_bo = NULL;
goto fail;
}
#endif
list_addtail(&to_vc4_bo(&exec->exec_bo->base)->unref_head,
&exec->unref_list);
exec->ct0ca = exec->exec_bo->paddr + bin_offset;
exec->bin_u = bin;
exec->shader_rec_v = exec->exec_bo->vaddr + shader_rec_offset;
exec->shader_rec_p = exec->exec_bo->paddr + shader_rec_offset;
exec->shader_rec_size = args->shader_rec_size;
exec->uniforms_v = exec->exec_bo->vaddr + uniforms_offset;
exec->uniforms_p = exec->exec_bo->paddr + uniforms_offset;
exec->uniforms_size = args->uniforms_size;
ret = vc4_validate_bin_cl(dev,
exec->exec_bo->vaddr + bin_offset,
bin,
exec);
if (ret)
goto fail;
ret = vc4_validate_shader_recs(dev, exec);
fail:
kfree(temp);
return ret;
}
