static int vc4_simulator_pin_bos(struct drm_device *dev, struct vc4_exec_info *exec) { struct drm_vc4_submit_cl *args = exec->args; struct vc4_context *vc4 = dev->vc4; struct vc4_bo **bos = vc4->bo_pointers.base; exec->bo_count = args->bo_handle_count; exec->bo = calloc(exec->bo_count, sizeof(void *)); for (int i = 0; i < exec->bo_count; i++) { struct vc4_bo *bo = bos[i]; struct drm_gem_cma_object *obj = vc4_wrap_bo_with_cma(dev, bo); struct drm_vc4_bo *drm_bo = to_vc4_bo(&obj->base); #if 0 fprintf(stderr, "bo hindex %d: %s\n", i, bo->name); #endif vc4_bo_map(bo); memcpy(obj->vaddr, bo->map, bo->size); exec->bo[i] = obj; /* The kernel does this validation at shader create ioctl * time. */ if (strcmp(bo->name, "code") == 0) { drm_bo->validated_shader = vc4_validate_shader(obj); if (!drm_bo->validated_shader) abort(); } } return 0; }
int vc4_mmap(struct file *filp, struct vm_area_struct *vma) { struct drm_gem_object *gem_obj; struct vc4_bo *bo; int ret; ret = drm_gem_mmap(filp, vma); if (ret) return ret; gem_obj = vma->vm_private_data; bo = to_vc4_bo(gem_obj); if (bo->validated_shader && (vma->vm_flags & VM_WRITE)) { DRM_ERROR("mmaping of shader BOs for writing not allowed.\n"); return -EINVAL; } /* * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map * the whole buffer. */ vma->vm_flags &= ~VM_PFNMAP; vma->vm_pgoff = 0; ret = dma_mmap_writecombine(bo->base.base.dev->dev, vma, bo->base.vaddr, bo->base.paddr, vma->vm_end - vma->vm_start); if (ret) drm_gem_vm_close(vma); return ret; }
void *vc4_prime_vmap(struct drm_gem_object *obj) { struct vc4_bo *bo = to_vc4_bo(obj); if (bo->validated_shader) { DRM_ERROR("mmaping of shader BOs not allowed.\n"); return ERR_PTR(-EINVAL); } return drm_gem_cma_prime_vmap(obj); }
int vc4_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) { struct vc4_bo *bo = to_vc4_bo(obj); if (bo->validated_shader && (vma->vm_flags & VM_WRITE)) { DRM_ERROR("mmaping of shader BOs for writing not allowed.\n"); return -EINVAL; } return drm_gem_cma_prime_mmap(obj, vma); }
struct dma_buf * vc4_prime_export(struct drm_device *dev, struct drm_gem_object *obj, int flags) { struct vc4_bo *bo = to_vc4_bo(obj); if (bo->validated_shader) { DRM_ERROR("Attempting to export shader BO\n"); return ERR_PTR(-EINVAL); } return drm_gem_prime_export(dev, obj, flags); }
/* Called on the last userspace/kernel unreference of the BO. Returns * it to the BO cache if possible, otherwise frees it. */ void vc4_free_object(struct drm_gem_object *gem_bo) { struct drm_device *dev = gem_bo->dev; struct vc4_dev *vc4 = to_vc4_dev(dev); struct vc4_bo *bo = to_vc4_bo(gem_bo); struct list_head *cache_list; mutex_lock(&vc4->bo_lock); /* If the object references someone else's memory, we can't cache it. */ if (gem_bo->import_attach) { vc4_bo_destroy(bo); goto out; } /* Don't cache if it was publicly named. */ if (gem_bo->name) { vc4_bo_destroy(bo); goto out; } /* If this object was partially constructed but CMA allocation * had failed, just free it. */ if (!bo->base.vaddr) { vc4_bo_destroy(bo); goto out; } cache_list = vc4_get_cache_list_for_size(dev, gem_bo->size); if (!cache_list) { vc4_bo_destroy(bo); goto out; } if (bo->validated_shader) { kfree(bo->validated_shader->texture_samples); kfree(bo->validated_shader); bo->validated_shader = NULL; } bo->free_time = jiffies; list_add(&bo->size_head, cache_list); list_add(&bo->unref_head, &vc4->bo_cache.time_list); vc4->bo_stats.num_cached++; vc4->bo_stats.size_cached += gem_bo->size; vc4_bo_cache_free_old(dev); out: mutex_unlock(&vc4->bo_lock); }
static int vc4_simulator_unpin_bos(struct vc4_exec_info *exec) { for (int i = 0; i < exec->bo_count; i++) { struct drm_gem_cma_object *obj = exec->bo[i]; struct drm_vc4_bo *drm_bo = to_vc4_bo(&obj->base); struct vc4_bo *bo = drm_bo->bo; assert(*(uint32_t *)(obj->vaddr + bo->size) == BO_SENTINEL); memcpy(bo->map, obj->vaddr, bo->size); if (drm_bo->validated_shader) { free(drm_bo->validated_shader->texture_samples); free(drm_bo->validated_shader); } free(obj); } free(exec->bo); return 0; }
struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t unaligned_size, bool allow_unzeroed) { size_t size = roundup(unaligned_size, PAGE_SIZE); struct vc4_dev *vc4 = to_vc4_dev(dev); struct drm_gem_cma_object *cma_obj; struct vc4_bo *bo; if (size == 0) return ERR_PTR(-EINVAL); /* First, try to get a vc4_bo from the kernel BO cache. */ bo = vc4_bo_get_from_cache(dev, size); if (bo) { if (!allow_unzeroed) memset(bo->base.vaddr, 0, bo->base.base.size); return bo; } cma_obj = drm_gem_cma_create(dev, size); if (IS_ERR(cma_obj)) { /* * If we've run out of CMA memory, kill the cache of * CMA allocations we've got laying around and try again. */ vc4_bo_cache_purge(dev); cma_obj = drm_gem_cma_create(dev, size); if (IS_ERR(cma_obj)) { DRM_ERROR("Failed to allocate from CMA:\n"); vc4_bo_stats_dump(vc4); return ERR_PTR(-ENOMEM); } } return to_vc4_bo(&cma_obj->base); }
static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec, struct vc4_rcl_setup *setup) { struct drm_vc4_submit_cl *args = exec->args; bool has_bin = args->bin_cl_size != 0; uint8_t min_x_tile = args->min_x_tile; uint8_t min_y_tile = args->min_y_tile; uint8_t max_x_tile = args->max_x_tile; uint8_t max_y_tile = args->max_y_tile; uint8_t xtiles = max_x_tile - min_x_tile + 1; uint8_t ytiles = max_y_tile - min_y_tile + 1; uint8_t x, y; uint32_t size, loop_body_size; size = VC4_PACKET_TILE_RENDERING_MODE_CONFIG_SIZE; loop_body_size = VC4_PACKET_TILE_COORDINATES_SIZE; if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) { size += VC4_PACKET_CLEAR_COLORS_SIZE + VC4_PACKET_TILE_COORDINATES_SIZE + VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE; } if (setup->color_read) { if (args->color_read.flags & VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) { loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE; } else { loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE; } } if (setup->zs_read) { if (args->zs_read.flags & VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) { loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE; } else { if (setup->color_read && !(args->color_read.flags & VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES)) { loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE; loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE; } loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE; } } if (has_bin) { size += VC4_PACKET_WAIT_ON_SEMAPHORE_SIZE; loop_body_size += VC4_PACKET_BRANCH_TO_SUB_LIST_SIZE; } if (setup->msaa_color_write) loop_body_size += VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE; if (setup->msaa_zs_write) loop_body_size += VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE; if (setup->zs_write) loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE; if (setup->color_write) loop_body_size += VC4_PACKET_STORE_MS_TILE_BUFFER_SIZE; /* We need a VC4_PACKET_TILE_COORDINATES in between each store. */ loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE * ((setup->msaa_color_write != NULL) + (setup->msaa_zs_write != NULL) + (setup->color_write != NULL) + (setup->zs_write != NULL) - 1); size += xtiles * ytiles * loop_body_size; setup->rcl = drm_gem_cma_create(dev, size); if (!setup->rcl) return -ENOMEM; list_addtail(&to_vc4_bo(&setup->rcl->base)->unref_head, &exec->unref_list); rcl_u8(setup, VC4_PACKET_TILE_RENDERING_MODE_CONFIG); rcl_u32(setup, (setup->color_write ? (setup->color_write->paddr + args->color_write.offset) : 0)); rcl_u16(setup, args->width); rcl_u16(setup, args->height); rcl_u16(setup, args->color_write.bits); /* The tile buffer gets cleared when the previous tile is stored. If * the clear values changed between frames, then the tile buffer has * stale clear values in it, so we have to do a store in None mode (no * writes) so that we trigger the tile buffer clear. */ if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) { rcl_u8(setup, VC4_PACKET_CLEAR_COLORS); rcl_u32(setup, args->clear_color[0]); rcl_u32(setup, args->clear_color[1]); rcl_u32(setup, args->clear_z); rcl_u8(setup, args->clear_s); vc4_tile_coordinates(setup, 0, 0); rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL); rcl_u16(setup, VC4_LOADSTORE_TILE_BUFFER_NONE); rcl_u32(setup, 0); /* no address, since we're in None mode */ } for (y = min_y_tile; y <= max_y_tile; y++) { for (x = min_x_tile; x <= max_x_tile; x++) { bool first = (x == min_x_tile && y == min_y_tile); bool last = (x == max_x_tile && y == max_y_tile); emit_tile(exec, setup, x, y, first, last); } } BUG_ON(setup->next_offset != size); exec->ct1ca = setup->rcl->paddr; exec->ct1ea = setup->rcl->paddr + setup->next_offset; return 0; }
/* * Copies in the user's binning command list and generates the validated bin * CL, along with associated data (shader records, uniforms). */ static int vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec) { struct drm_vc4_submit_cl *args = exec->args; void *temp = NULL; void *bin; int ret = 0; uint32_t bin_offset = 0; uint32_t shader_rec_offset = roundup(bin_offset + args->bin_cl_size, 16); uint32_t uniforms_offset = shader_rec_offset + args->shader_rec_size; uint32_t exec_size = uniforms_offset + args->uniforms_size; uint32_t temp_size = exec_size + (sizeof(struct vc4_shader_state) * args->shader_rec_count); if (uniforms_offset < shader_rec_offset || exec_size < uniforms_offset || args->shader_rec_count >= (UINT_MAX / sizeof(struct vc4_shader_state)) || temp_size < exec_size) { DRM_ERROR("overflow in exec arguments\n"); goto fail; } /* Allocate space where we'll store the copied in user command lists * and shader records. * * We don't just copy directly into the BOs because we need to * read the contents back for validation, and I think the * bo->vaddr is uncached access. */ temp = kmalloc(temp_size, GFP_KERNEL); if (!temp) { DRM_ERROR("Failed to allocate storage for copying " "in bin/render CLs.\n"); ret = -ENOMEM; goto fail; } bin = temp + bin_offset; exec->shader_rec_u = temp + shader_rec_offset; exec->uniforms_u = temp + uniforms_offset; exec->shader_state = temp + exec_size; exec->shader_state_size = args->shader_rec_count; ret = copy_from_user(bin, (void __user *)(uintptr_t)args->bin_cl, args->bin_cl_size); if (ret) { DRM_ERROR("Failed to copy in bin cl\n"); goto fail; } ret = copy_from_user(exec->shader_rec_u, (void __user *)(uintptr_t)args->shader_rec, args->shader_rec_size); if (ret) { DRM_ERROR("Failed to copy in shader recs\n"); goto fail; } ret = copy_from_user(exec->uniforms_u, (void __user *)(uintptr_t)args->uniforms, args->uniforms_size); if (ret) { DRM_ERROR("Failed to copy in uniforms cl\n"); goto fail; } exec->exec_bo = drm_gem_cma_create(dev, exec_size); #if 0 if (IS_ERR(exec->exec_bo)) { DRM_ERROR("Couldn't allocate BO for exec\n"); ret = PTR_ERR(exec->exec_bo); exec->exec_bo = NULL; goto fail; } #endif list_addtail(&to_vc4_bo(&exec->exec_bo->base)->unref_head, &exec->unref_list); exec->ct0ca = exec->exec_bo->paddr + bin_offset; exec->bin_u = bin; exec->shader_rec_v = exec->exec_bo->vaddr + shader_rec_offset; exec->shader_rec_p = exec->exec_bo->paddr + shader_rec_offset; exec->shader_rec_size = args->shader_rec_size; exec->uniforms_v = exec->exec_bo->vaddr + uniforms_offset; exec->uniforms_p = exec->exec_bo->paddr + uniforms_offset; exec->uniforms_size = args->uniforms_size; ret = vc4_validate_bin_cl(dev, exec->exec_bo->vaddr + bin_offset, bin, exec); if (ret) goto fail; ret = vc4_validate_shader_recs(dev, exec); fail: kfree(temp); return ret; }