LOCAL cl_mem cl_mem_new_libva_buffer(cl_context ctx,
unsigned int bo_name,
cl_int* errcode)
{
cl_int err = CL_SUCCESS;
cl_mem mem = NULL;
mem = cl_mem_allocate(CL_MEM_BUFFER_TYPE, ctx, 0, 0, CL_FALSE, &err);
if (mem == NULL || err != CL_SUCCESS)
goto error;
size_t sz = 0;
mem->bo = cl_buffer_get_buffer_from_libva(ctx, bo_name, &sz);
mem->size = sz;
exit:
if (errcode)
*errcode = err;
return mem;
error:
cl_mem_delete(mem);
mem = NULL;
goto exit;
}
LOCAL cl_mem cl_mem_new_libva_image(cl_context ctx,
unsigned int bo_name, size_t offset,
size_t width, size_t height,
cl_image_format fmt,
size_t row_pitch,
cl_int *errcode)
{
cl_int err = CL_SUCCESS;
cl_mem mem = NULL;
struct _cl_mem_image *image = NULL;
uint32_t intel_fmt, bpp;
intel_fmt = cl_image_get_intel_format(&fmt);
if (intel_fmt == INTEL_UNSUPPORTED_FORMAT) {
err = CL_IMAGE_FORMAT_NOT_SUPPORTED;
goto error;
}
cl_image_byte_per_pixel(&fmt, &bpp);
mem = cl_mem_allocate(CL_MEM_IMAGE_TYPE, ctx, 0, 0, 0, &err);
if (mem == NULL || err != CL_SUCCESS) {
err = CL_OUT_OF_HOST_MEMORY;
goto error;
}
image = cl_mem_image(mem);
mem->bo = cl_buffer_get_image_from_libva(ctx, bo_name, image);
image->w = width;
image->h = height;
image->image_type = CL_MEM_OBJECT_IMAGE2D;
image->depth = 2;
image->fmt = fmt;
image->intel_fmt = intel_fmt;
image->bpp = bpp;
image->row_pitch = row_pitch;
image->slice_pitch = 0;
// NOTE: tiling of image is set in cl_buffer_get_image_from_libva().
image->tile_x = 0;
image->tile_y = 0;
image->offset = offset;
exit:
if (errcode)
*errcode = err;
return mem;
error:
cl_mem_delete(mem);
mem = NULL;
goto exit;
}
LOCAL cl_mem
cl_mem_new_gl_texture(cl_context ctx,
cl_mem_flags flags,
GLenum texture_target,
GLint miplevel,
GLuint texture,
cl_int *errcode_ret)
{
cl_int err = CL_SUCCESS;
cl_mem mem = NULL;
/* Check flags consistency */
if (UNLIKELY(flags & CL_MEM_COPY_HOST_PTR)) {
err = CL_INVALID_ARG_VALUE;
goto error;
}
mem = cl_mem_allocate(CL_MEM_GL_IMAGE_TYPE, ctx, flags, 0, 0, NULL, &err);
if (mem == NULL || err != CL_SUCCESS)
goto error;
mem->bo = cl_buffer_alloc_from_texture(ctx, texture_target, miplevel,
texture, cl_mem_image(mem));
if (UNLIKELY(mem->bo == NULL)) {
err = CL_MEM_OBJECT_ALLOCATION_FAILURE;
goto error;
}
cl_mem_gl_image(mem)->target = texture_target;
cl_mem_gl_image(mem)->miplevel = miplevel;
cl_mem_gl_image(mem)->texture = texture;
exit:
if (errcode_ret)
*errcode_ret = err;
return mem;
error:
cl_mem_delete(mem);
mem = NULL;
goto exit;
}
LOCAL void
cl_mem_delete(cl_mem mem)
{
cl_int i;
if (UNLIKELY(mem == NULL))
return;
if (atomic_dec(&mem->ref_n) > 1)
return;
#ifdef HAS_EGL
if (UNLIKELY(IS_GL_IMAGE(mem))) {
cl_mem_gl_delete(cl_mem_gl_image(mem));
}
#endif
/* Remove it from the list */
assert(mem->ctx);
pthread_mutex_lock(&mem->ctx->buffer_lock);
if (mem->prev)
mem->prev->next = mem->next;
if (mem->next)
mem->next->prev = mem->prev;
if (mem->ctx->buffers == mem)
mem->ctx->buffers = mem->next;
pthread_mutex_unlock(&mem->ctx->buffer_lock);
cl_context_delete(mem->ctx);
/* Someone still mapped, unmap */
if(mem->map_ref > 0) {
assert(mem->mapped_ptr);
for(i=0; i<mem->mapped_ptr_sz; i++) {
if(mem->mapped_ptr[i].ptr != NULL) {
mem->map_ref--;
cl_mem_unmap_gtt(mem);
}
}
assert(mem->map_ref == 0);
}
if (mem->mapped_ptr)
free(mem->mapped_ptr);
if (mem->dstr_cb) {
cl_mem_dstr_cb *cb = mem->dstr_cb;
while (mem->dstr_cb) {
cb = mem->dstr_cb;
cb->pfn_notify(mem, cb->user_data);
mem->dstr_cb = cb->next;
free(cb);
}
}
/* Iff we are sub, do nothing for bo release. */
if (mem->type == CL_MEM_SUBBUFFER_TYPE) {
struct _cl_mem_buffer* buffer = (struct _cl_mem_buffer*)mem;
/* Remove it from the parent's list */
assert(buffer->parent);
pthread_mutex_lock(&buffer->parent->sub_lock);
if (buffer->sub_prev)
buffer->sub_prev->sub_next = buffer->sub_next;
if (buffer->sub_next)
buffer->sub_next->sub_prev = buffer->sub_prev;
if (buffer->parent->subs == buffer)
buffer->parent->subs = buffer->sub_next;
pthread_mutex_unlock(&buffer->parent->sub_lock);
cl_mem_delete((cl_mem )(buffer->parent));
} else if (LIKELY(mem->bo != NULL)) {
cl_buffer_unreference(mem->bo);
}
cl_free(mem);
}
static cl_mem
_cl_mem_new_image(cl_context ctx,
cl_mem_flags flags,
const cl_image_format *fmt,
const cl_mem_object_type image_type,
size_t w,
size_t h,
size_t depth,
size_t pitch,
size_t slice_pitch,
void *data,
cl_int *errcode_ret)
{
cl_int err = CL_SUCCESS;
cl_mem mem = NULL;
uint32_t bpp = 0, intel_fmt = INTEL_UNSUPPORTED_FORMAT;
size_t sz = 0, aligned_pitch = 0, aligned_slice_pitch = 0, aligned_h;
cl_image_tiling_t tiling = CL_NO_TILE;
/* Check flags consistency */
if (UNLIKELY((flags & (CL_MEM_COPY_HOST_PTR | CL_MEM_USE_HOST_PTR)) && data == NULL)) {
err = CL_INVALID_HOST_PTR;
goto error;
}
/* Get the size of each pixel */
if (UNLIKELY((err = cl_image_byte_per_pixel(fmt, &bpp)) != CL_SUCCESS))
goto error;
/* Only a sub-set of the formats are supported */
intel_fmt = cl_image_get_intel_format(fmt);
if (UNLIKELY(intel_fmt == INTEL_UNSUPPORTED_FORMAT)) {
err = CL_INVALID_IMAGE_FORMAT_DESCRIPTOR;
goto error;
}
/* See if the user parameters match */
#define DO_IMAGE_ERROR \
do { \
err = CL_INVALID_IMAGE_SIZE; \
goto error; \
} while (0);
if (UNLIKELY(w == 0)) DO_IMAGE_ERROR;
if (UNLIKELY(h == 0)) DO_IMAGE_ERROR;
if (image_type == CL_MEM_OBJECT_IMAGE2D) {
size_t min_pitch = bpp * w;
if (data && pitch == 0)
pitch = min_pitch;
if (UNLIKELY(w > ctx->device->image2d_max_width)) DO_IMAGE_ERROR;
if (UNLIKELY(h > ctx->device->image2d_max_height)) DO_IMAGE_ERROR;
if (UNLIKELY(data && min_pitch > pitch)) DO_IMAGE_ERROR;
if (UNLIKELY(!data && pitch != 0)) DO_IMAGE_ERROR;
/* Pick up tiling mode (we do only linear on SNB) */
if (cl_driver_get_ver(ctx->drv) != 6)
tiling = CL_TILE_Y;
depth = 1;
}
if (image_type == CL_MEM_OBJECT_IMAGE3D) {
size_t min_pitch = bpp * w;
if (data && pitch == 0)
pitch = min_pitch;
size_t min_slice_pitch = min_pitch * h;
if (data && slice_pitch == 0)
slice_pitch = min_slice_pitch;
if (UNLIKELY(w > ctx->device->image3d_max_width)) DO_IMAGE_ERROR;
if (UNLIKELY(h > ctx->device->image3d_max_height)) DO_IMAGE_ERROR;
if (UNLIKELY(depth > ctx->device->image3d_max_depth)) DO_IMAGE_ERROR;
if (UNLIKELY(data && min_pitch > pitch)) DO_IMAGE_ERROR;
if (UNLIKELY(data && min_slice_pitch > slice_pitch)) DO_IMAGE_ERROR;
if (UNLIKELY(!data && pitch != 0)) DO_IMAGE_ERROR;
if (UNLIKELY(!data && slice_pitch != 0)) DO_IMAGE_ERROR;
/* Pick up tiling mode (we do only linear on SNB) */
if (cl_driver_get_ver(ctx->drv) != 6)
tiling = CL_TILE_Y;
}
#undef DO_IMAGE_ERROR
/* Tiling requires to align both pitch and height */
if (tiling == CL_NO_TILE) {
aligned_pitch = w * bpp;
aligned_h = h;
} else if (tiling == CL_TILE_X) {
aligned_pitch = ALIGN(w * bpp, tilex_w);
aligned_h = ALIGN(h, tilex_h);
} else if (tiling == CL_TILE_Y) {
aligned_pitch = ALIGN(w * bpp, tiley_w);
aligned_h = ALIGN(h, tiley_h);
}
sz = aligned_pitch * aligned_h * depth;
mem = cl_mem_allocate(CL_MEM_IMAGE_TYPE, ctx, flags, sz, tiling != CL_NO_TILE, &err);
if (mem == NULL || err != CL_SUCCESS)
goto error;
cl_buffer_set_tiling(mem->bo, tiling, aligned_pitch);
aligned_slice_pitch = (image_type == CL_MEM_OBJECT_IMAGE1D
|| image_type == CL_MEM_OBJECT_IMAGE2D) ? 0 : aligned_pitch * ALIGN(h, 2);
cl_mem_image_init(cl_mem_image(mem), w, h, image_type, depth, *fmt,
intel_fmt, bpp, aligned_pitch, aligned_slice_pitch, tiling,
0, 0, 0);
/* Copy the data if required */
if (flags & (CL_MEM_COPY_HOST_PTR | CL_MEM_USE_HOST_PTR)) {
cl_mem_copy_image(cl_mem_image(mem), pitch, slice_pitch, data);
if (flags & CL_MEM_USE_HOST_PTR) {
mem->host_ptr = data;
cl_mem_image(mem)->host_row_pitch = pitch;
cl_mem_image(mem)->host_slice_pitch = slice_pitch;
}
}
exit:
if (errcode_ret)
*errcode_ret = err;
return mem;
error:
cl_mem_delete(mem);
mem = NULL;
goto exit;
}
LOCAL cl_mem
cl_mem_new_sub_buffer(cl_mem buffer,
cl_mem_flags flags,
cl_buffer_create_type create_type,
const void *create_info,
cl_int *errcode_ret)
{
cl_int err = CL_SUCCESS;
cl_mem mem = NULL;
struct _cl_mem_buffer *sub_buf = NULL;
if (buffer->type != CL_MEM_BUFFER_TYPE) {
err = CL_INVALID_MEM_OBJECT;
goto error;
}
if (flags && (((buffer->flags & CL_MEM_WRITE_ONLY) && (flags & (CL_MEM_READ_WRITE|CL_MEM_READ_ONLY)))
|| ((buffer->flags & CL_MEM_READ_ONLY) && (flags & (CL_MEM_READ_WRITE|CL_MEM_WRITE_ONLY)))
|| (flags & (CL_MEM_USE_HOST_PTR | CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR)))) {
err = CL_INVALID_VALUE;
goto error;
}
if (create_type != CL_BUFFER_CREATE_TYPE_REGION) {
err = CL_INVALID_VALUE;
goto error;
}
if (!create_info) {
err = CL_INVALID_VALUE;
goto error;
}
cl_buffer_region *info = (cl_buffer_region *)create_info;
if (!info->size) {
err = CL_INVALID_BUFFER_SIZE;
goto error;
}
if (info->origin > buffer->size || info->origin + info->size > buffer->size) {
err = CL_INVALID_VALUE;
goto error;
}
if (info->origin & (buffer->ctx->device->mem_base_addr_align - 1)) {
err = CL_MISALIGNED_SUB_BUFFER_OFFSET;
goto error;
}
/* Now create the sub buffer and link it to the buffer. */
TRY_ALLOC (sub_buf, CALLOC(struct _cl_mem_buffer));
mem = &sub_buf->base;
mem->type = CL_MEM_SUBBUFFER_TYPE;
SET_ICD(mem->dispatch)
mem->ref_n = 1;
mem->magic = CL_MAGIC_MEM_HEADER;
mem->flags = flags;
sub_buf->parent = (struct _cl_mem_buffer*)buffer;
cl_mem_add_ref(buffer);
/* Append the buffer in the parent buffer list */
pthread_mutex_lock(&((struct _cl_mem_buffer*)buffer)->sub_lock);
sub_buf->sub_next = ((struct _cl_mem_buffer*)buffer)->subs;
if (((struct _cl_mem_buffer*)buffer)->subs != NULL)
((struct _cl_mem_buffer*)buffer)->subs->sub_prev = sub_buf;
((struct _cl_mem_buffer*)buffer)->subs = sub_buf;
pthread_mutex_unlock(&((struct _cl_mem_buffer*)buffer)->sub_lock);
mem->bo = buffer->bo;
mem->size = info->size;
sub_buf->sub_offset = info->origin;
if (buffer->flags & CL_MEM_USE_HOST_PTR || buffer->flags & CL_MEM_COPY_HOST_PTR) {
mem->host_ptr = buffer->host_ptr;
}
cl_context_add_ref(buffer->ctx);
mem->ctx = buffer->ctx;
/* Append the buffer in the context buffer list */
pthread_mutex_lock(&buffer->ctx->buffer_lock);
mem->next = buffer->ctx->buffers;
if (buffer->ctx->buffers != NULL)
buffer->ctx->buffers->prev = mem;
buffer->ctx->buffers = mem;
pthread_mutex_unlock(&buffer->ctx->buffer_lock);
exit:
if (errcode_ret)
*errcode_ret = err;
return mem;
error:
cl_mem_delete(mem);
mem = NULL;
goto exit;
}
LOCAL cl_mem
cl_mem_new_buffer(cl_context ctx,
cl_mem_flags flags,
size_t sz,
void *data,
cl_int *errcode_ret)
{
/* Possible mem type combination:
CL_MEM_ALLOC_HOST_PTR
CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR
CL_MEM_USE_HOST_PTR
CL_MEM_COPY_HOST_PTR */
cl_int err = CL_SUCCESS;
cl_mem mem = NULL;
cl_ulong max_mem_size;
if (UNLIKELY(sz == 0)) {
err = CL_INVALID_BUFFER_SIZE;
goto error;
}
if (UNLIKELY(((flags & CL_MEM_READ_WRITE)
&& (flags & (CL_MEM_READ_ONLY | CL_MEM_WRITE_ONLY)))
|| ((flags & CL_MEM_READ_ONLY) && (flags & (CL_MEM_WRITE_ONLY)))
|| ((flags & CL_MEM_ALLOC_HOST_PTR) && (flags & CL_MEM_USE_HOST_PTR))
|| ((flags & CL_MEM_COPY_HOST_PTR) && (flags & CL_MEM_USE_HOST_PTR))
|| ((flags & (~(CL_MEM_READ_WRITE | CL_MEM_WRITE_ONLY | CL_MEM_READ_ONLY
| CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR
| CL_MEM_USE_HOST_PTR))) != 0))) {
err = CL_INVALID_VALUE;
goto error;
}
/* This flag is valid only if host_ptr is not NULL */
if (UNLIKELY((((flags & CL_MEM_COPY_HOST_PTR) ||
(flags & CL_MEM_USE_HOST_PTR)) &&
data == NULL))
|| (!(flags & (CL_MEM_COPY_HOST_PTR
|CL_MEM_USE_HOST_PTR))
&& (data != NULL))) {
err = CL_INVALID_HOST_PTR;
goto error;
}
/* CL_MEM_ALLOC_HOST_PTR and CL_MEM_USE_HOST_PTR
are mutually exclusive. */
if (UNLIKELY(flags & CL_MEM_ALLOC_HOST_PTR &&
flags & CL_MEM_USE_HOST_PTR)) {
err = CL_INVALID_HOST_PTR;
goto error;
}
/* CL_MEM_COPY_HOST_PTR and CL_MEM_USE_HOST_PTR
are mutually exclusive. */
if (UNLIKELY(flags & CL_MEM_COPY_HOST_PTR &&
flags & CL_MEM_USE_HOST_PTR)) {
err = CL_INVALID_HOST_PTR;
goto error;
}
if ((err = cl_get_device_info(ctx->device,
CL_DEVICE_MAX_MEM_ALLOC_SIZE,
sizeof(max_mem_size),
&max_mem_size,
NULL)) != CL_SUCCESS) {
goto error;
}
if (UNLIKELY(sz > max_mem_size)) {
err = CL_INVALID_BUFFER_SIZE;
goto error;
}
/* Create the buffer in video memory */
mem = cl_mem_allocate(CL_MEM_BUFFER_TYPE, ctx, flags, sz, CL_FALSE, &err);
if (mem == NULL || err != CL_SUCCESS)
goto error;
/* Copy the data if required */
if (flags & CL_MEM_COPY_HOST_PTR || flags & CL_MEM_USE_HOST_PTR)
cl_buffer_subdata(mem->bo, 0, sz, data);
if (flags & CL_MEM_USE_HOST_PTR || flags & CL_MEM_COPY_HOST_PTR)
mem->host_ptr = data;
exit:
if (errcode_ret)
*errcode_ret = err;
return mem;
error:
cl_mem_delete(mem);
mem = NULL;
goto exit;
}
LOCAL cl_mem
cl_mem_allocate(enum cl_mem_type type,
cl_context ctx,
cl_mem_flags flags,
size_t sz,
cl_int is_tiled,
cl_int *errcode)
{
cl_buffer_mgr bufmgr = NULL;
cl_mem mem = NULL;
cl_int err = CL_SUCCESS;
size_t alignment = 64;
assert(ctx);
/* Allocate and inialize the structure itself */
if (type == CL_MEM_IMAGE_TYPE) {
struct _cl_mem_image *image = NULL;
TRY_ALLOC (image, CALLOC(struct _cl_mem_image));
mem = &image->base;
} else if (type == CL_MEM_GL_IMAGE_TYPE ) {
struct _cl_mem_gl_image *gl_image = NULL;
TRY_ALLOC (gl_image, CALLOC(struct _cl_mem_gl_image));
mem = &gl_image->base.base;
} else {
struct _cl_mem_buffer *buffer = NULL;
TRY_ALLOC (buffer, CALLOC(struct _cl_mem_buffer));
mem = &buffer->base;
}
mem->type = type;
SET_ICD(mem->dispatch)
mem->ref_n = 1;
mem->magic = CL_MAGIC_MEM_HEADER;
mem->flags = flags;
if (sz != 0) {
/* Pinning will require stricter alignment rules */
if ((flags & CL_MEM_PINNABLE) || is_tiled)
alignment = 4096;
/* Allocate space in memory */
bufmgr = cl_context_get_bufmgr(ctx);
assert(bufmgr);
mem->bo = cl_buffer_alloc(bufmgr, "CL memory object", sz, alignment);
if (UNLIKELY(mem->bo == NULL)) {
err = CL_MEM_OBJECT_ALLOCATION_FAILURE;
goto error;
}
mem->size = sz;
}
cl_context_add_ref(ctx);
mem->ctx = ctx;
/* Append the buffer in the context buffer list */
pthread_mutex_lock(&ctx->buffer_lock);
mem->next = ctx->buffers;
if (ctx->buffers != NULL)
ctx->buffers->prev = mem;
ctx->buffers = mem;
pthread_mutex_unlock(&ctx->buffer_lock);
exit:
if (errcode)
*errcode = err;
return mem;
error:
cl_mem_delete(mem);
mem = NULL;
goto exit;
}
