/**
* @brief lp_build_fetch_rgba_aos_array
*
* \param format_desc describes format of the image we're fetching from
* \param dst_type output type
* \param base_ptr address of the pixel block (or the texel if uncompressed)
* \param offset ptr offset
*/
LLVMValueRef
lp_build_fetch_rgba_aos_array(struct gallivm_state *gallivm,
const struct util_format_description *format_desc,
struct lp_type dst_type,
LLVMValueRef base_ptr,
LLVMValueRef offset)
{
struct lp_build_context bld;
LLVMBuilderRef builder = gallivm->builder;
LLVMTypeRef src_elem_type, src_vec_type;
LLVMValueRef ptr, res = NULL;
struct lp_type src_type;
memset(&src_type, 0, sizeof src_type);
src_type.floating = format_desc->channel[0].type == UTIL_FORMAT_TYPE_FLOAT;
src_type.fixed = format_desc->channel[0].type == UTIL_FORMAT_TYPE_FIXED;
src_type.sign = format_desc->channel[0].type != UTIL_FORMAT_TYPE_UNSIGNED;
src_type.norm = format_desc->channel[0].normalized;
src_type.width = format_desc->channel[0].size;
src_type.length = format_desc->nr_channels;
assert(src_type.length <= dst_type.length);
src_elem_type = lp_build_elem_type(gallivm, src_type);
src_vec_type = lp_build_vec_type(gallivm, src_type);
/* Read whole vector from memory, unaligned */
if (!res) {
ptr = LLVMBuildGEP(builder, base_ptr, &offset, 1, "");
ptr = LLVMBuildPointerCast(builder, ptr, LLVMPointerType(src_vec_type, 0), "");
res = LLVMBuildLoad(builder, ptr, "");
lp_set_load_alignment(res, src_type.width / 8);
}
/* Truncate doubles to float */
if (src_type.floating && src_type.width == 64) {
src_type.width = 32;
src_vec_type = lp_build_vec_type(gallivm, src_type);
res = LLVMBuildFPTrunc(builder, res, src_vec_type, "");
}
/* Expand to correct length */
if (src_type.length < dst_type.length) {
res = lp_build_pad_vector(gallivm, res, src_type, dst_type.length);
src_type.length = dst_type.length;
}
/* Convert to correct format */
lp_build_conv(gallivm, src_type, dst_type, &res, 1, &res, 1);
/* Swizzle it */
lp_build_context_init(&bld, gallivm, dst_type);
return lp_build_format_swizzle_aos(format_desc, &bld, res);
}
/**
* Fetch a pixel into a 4 float AoS.
*
* \param format_desc describes format of the image we're fetching from
* \param ptr address of the pixel block (or the texel if uncompressed)
* \param i, j the sub-block pixel coordinates. For non-compressed formats
* these will always be (0, 0).
* \return a 4 element vector with the pixel's RGBA values.
*/
LLVMValueRef
lp_build_fetch_rgba_aos(struct gallivm_state *gallivm,
const struct util_format_description *format_desc,
struct lp_type type,
LLVMValueRef base_ptr,
LLVMValueRef offset,
LLVMValueRef i,
LLVMValueRef j)
{
LLVMBuilderRef builder = gallivm->builder;
unsigned num_pixels = type.length / 4;
struct lp_build_context bld;
assert(type.length <= LP_MAX_VECTOR_LENGTH);
assert(type.length % 4 == 0);
lp_build_context_init(&bld, gallivm, type);
/*
* Trivial case
*
* The format matches the type (apart of a swizzle) so no need for
* scaling or converting.
*/
if (format_matches_type(format_desc, type) &&
format_desc->block.bits <= type.width * 4 &&
util_is_power_of_two(format_desc->block.bits)) {
LLVMValueRef packed;
/*
* The format matches the type (apart of a swizzle) so no need for
* scaling or converting.
*/
packed = lp_build_gather(gallivm, type.length/4,
format_desc->block.bits, type.width*4,
base_ptr, offset);
assert(format_desc->block.bits <= type.width * type.length);
packed = LLVMBuildBitCast(gallivm->builder, packed,
lp_build_vec_type(gallivm, type), "");
return lp_build_format_swizzle_aos(format_desc, &bld, packed);
}
/*
* Bit arithmetic
*/
if (format_desc->layout == UTIL_FORMAT_LAYOUT_PLAIN &&
(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB ||
format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS) &&
format_desc->block.width == 1 &&
format_desc->block.height == 1 &&
util_is_power_of_two(format_desc->block.bits) &&
format_desc->block.bits <= 32 &&
format_desc->is_bitmask &&
!format_desc->is_mixed &&
(format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED ||
format_desc->channel[1].type == UTIL_FORMAT_TYPE_UNSIGNED)) {
LLVMValueRef tmps[LP_MAX_VECTOR_LENGTH/4];
LLVMValueRef res;
unsigned k;
/*
* Unpack a pixel at a time into a <4 x float> RGBA vector
*/
for (k = 0; k < num_pixels; ++k) {
LLVMValueRef packed;
packed = lp_build_gather_elem(gallivm, num_pixels,
format_desc->block.bits, 32,
base_ptr, offset, k);
tmps[k] = lp_build_unpack_arith_rgba_aos(gallivm,
format_desc,
packed);
}
/*
* Type conversion.
*
* TODO: We could avoid floating conversion for integer to
* integer conversions.
*/
if (gallivm_debug & GALLIVM_DEBUG_PERF && !type.floating) {
debug_printf("%s: unpacking %s with floating point\n",
__FUNCTION__, format_desc->short_name);
}
lp_build_conv(gallivm,
lp_float32_vec4_type(),
type,
tmps, num_pixels, &res, 1);
return lp_build_format_swizzle_aos(format_desc, &bld, res);
}
/*
* YUV / subsampled formats
*/
if (format_desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED) {
struct lp_type tmp_type;
LLVMValueRef tmp;
memset(&tmp_type, 0, sizeof tmp_type);
tmp_type.width = 8;
tmp_type.length = num_pixels * 4;
tmp_type.norm = TRUE;
tmp = lp_build_fetch_subsampled_rgba_aos(gallivm,
format_desc,
num_pixels,
base_ptr,
offset,
i, j);
lp_build_conv(gallivm,
tmp_type, type,
&tmp, 1, &tmp, 1);
return tmp;
}
/*
* Fallback to util_format_description::fetch_rgba_8unorm().
*/
if (format_desc->fetch_rgba_8unorm &&
!type.floating && type.width == 8 && !type.sign && type.norm) {
/*
* Fallback to calling util_format_description::fetch_rgba_8unorm.
*
* This is definitely not the most efficient way of fetching pixels, as
* we miss the opportunity to do vectorization, but this it is a
* convenient for formats or scenarios for which there was no opportunity
* or incentive to optimize.
*/
LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(gallivm->builder)));
char name[256];
LLVMTypeRef i8t = LLVMInt8TypeInContext(gallivm->context);
LLVMTypeRef pi8t = LLVMPointerType(i8t, 0);
LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
LLVMValueRef function;
LLVMValueRef tmp_ptr;
LLVMValueRef tmp;
LLVMValueRef res;
LLVMValueRef callee;
unsigned k;
util_snprintf(name, sizeof name, "util_format_%s_fetch_rgba_8unorm",
format_desc->short_name);
if (gallivm_debug & GALLIVM_DEBUG_PERF) {
debug_printf("%s: falling back to %s\n", __FUNCTION__, name);
}
/*
* Declare and bind format_desc->fetch_rgba_8unorm().
*/
function = LLVMGetNamedFunction(module, name);
if (!function) {
/*
* Function to call looks like:
* fetch(uint8_t *dst, const uint8_t *src, unsigned i, unsigned j)
*/
LLVMTypeRef ret_type;
LLVMTypeRef arg_types[4];
LLVMTypeRef function_type;
ret_type = LLVMVoidTypeInContext(gallivm->context);
arg_types[0] = pi8t;
arg_types[1] = pi8t;
arg_types[2] = i32t;
arg_types[3] = i32t;
function_type = LLVMFunctionType(ret_type, arg_types,
Elements(arg_types), 0);
function = LLVMAddFunction(module, name, function_type);
LLVMSetFunctionCallConv(function, LLVMCCallConv);
LLVMSetLinkage(function, LLVMExternalLinkage);
assert(LLVMIsDeclaration(function));
}
/* make const pointer for the C fetch_rgba_float function */
callee = lp_build_const_int_pointer(gallivm,
func_to_pointer((func_pointer) format_desc->fetch_rgba_8unorm));
/* cast the callee pointer to the function's type */
function = LLVMBuildBitCast(builder, callee,
LLVMTypeOf(function), "cast callee");
tmp_ptr = lp_build_alloca(gallivm, i32t, "");
res = LLVMGetUndef(LLVMVectorType(i32t, num_pixels));
/*
* Invoke format_desc->fetch_rgba_8unorm() for each pixel and insert the result
* in the SoA vectors.
*/
for (k = 0; k < num_pixels; ++k) {
LLVMValueRef index = lp_build_const_int32(gallivm, k);
LLVMValueRef args[4];
args[0] = LLVMBuildBitCast(builder, tmp_ptr, pi8t, "");
args[1] = lp_build_gather_elem_ptr(gallivm, num_pixels,
base_ptr, offset, k);
if (num_pixels == 1) {
args[2] = i;
args[3] = j;
}
else {
args[2] = LLVMBuildExtractElement(builder, i, index, "");
args[3] = LLVMBuildExtractElement(builder, j, index, "");
}
LLVMBuildCall(builder, function, args, Elements(args), "");
tmp = LLVMBuildLoad(builder, tmp_ptr, "");
if (num_pixels == 1) {
res = tmp;
}
else {
res = LLVMBuildInsertElement(builder, res, tmp, index, "");
}
}
/* Bitcast from <n x i32> to <4n x i8> */
res = LLVMBuildBitCast(builder, res, bld.vec_type, "");
return res;
}
/*
* Fallback to util_format_description::fetch_rgba_float().
*/
if (format_desc->fetch_rgba_float) {
/*
* Fallback to calling util_format_description::fetch_rgba_float.
*
* This is definitely not the most efficient way of fetching pixels, as
* we miss the opportunity to do vectorization, but this it is a
* convenient for formats or scenarios for which there was no opportunity
* or incentive to optimize.
*/
LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
char name[256];
LLVMTypeRef f32t = LLVMFloatTypeInContext(gallivm->context);
LLVMTypeRef f32x4t = LLVMVectorType(f32t, 4);
LLVMTypeRef pf32t = LLVMPointerType(f32t, 0);
LLVMTypeRef pi8t = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
LLVMValueRef function;
LLVMValueRef tmp_ptr;
LLVMValueRef tmps[LP_MAX_VECTOR_LENGTH/4];
LLVMValueRef res;
LLVMValueRef callee;
unsigned k;
util_snprintf(name, sizeof name, "util_format_%s_fetch_rgba_float",
format_desc->short_name);
if (gallivm_debug & GALLIVM_DEBUG_PERF) {
debug_printf("%s: falling back to %s\n", __FUNCTION__, name);
}
/*
* Declare and bind format_desc->fetch_rgba_float().
*/
function = LLVMGetNamedFunction(module, name);
if (!function) {
/*
* Function to call looks like:
* fetch(float *dst, const uint8_t *src, unsigned i, unsigned j)
*/
LLVMTypeRef ret_type;
LLVMTypeRef arg_types[4];
LLVMTypeRef function_type;
ret_type = LLVMVoidTypeInContext(gallivm->context);
arg_types[0] = pf32t;
arg_types[1] = pi8t;
arg_types[2] = i32t;
arg_types[3] = i32t;
function_type = LLVMFunctionType(ret_type, arg_types,
Elements(arg_types), 0);
function = LLVMAddFunction(module, name, function_type);
LLVMSetFunctionCallConv(function, LLVMCCallConv);
LLVMSetLinkage(function, LLVMExternalLinkage);
assert(LLVMIsDeclaration(function));
}
/* Note: we're using this casting here instead of LLVMAddGlobalMapping()
* to work around a bug in LLVM 2.6.
*/
/* make const pointer for the C fetch_rgba_float function */
callee = lp_build_const_int_pointer(gallivm,
func_to_pointer((func_pointer) format_desc->fetch_rgba_float));
/* cast the callee pointer to the function's type */
function = LLVMBuildBitCast(builder, callee,
LLVMTypeOf(function), "cast callee");
tmp_ptr = lp_build_alloca(gallivm, f32x4t, "");
/*
* Invoke format_desc->fetch_rgba_float() for each pixel and insert the result
* in the SoA vectors.
*/
for (k = 0; k < num_pixels; ++k) {
LLVMValueRef args[4];
args[0] = LLVMBuildBitCast(builder, tmp_ptr, pf32t, "");
args[1] = lp_build_gather_elem_ptr(gallivm, num_pixels,
base_ptr, offset, k);
if (num_pixels == 1) {
args[2] = i;
args[3] = j;
}
else {
LLVMValueRef index = lp_build_const_int32(gallivm, k);
args[2] = LLVMBuildExtractElement(builder, i, index, "");
args[3] = LLVMBuildExtractElement(builder, j, index, "");
}
LLVMBuildCall(builder, function, args, Elements(args), "");
tmps[k] = LLVMBuildLoad(builder, tmp_ptr, "");
}
lp_build_conv(gallivm,
lp_float32_vec4_type(),
type,
tmps, num_pixels, &res, 1);
return res;
}
assert(0);
return lp_build_undef(gallivm, type);
}
/**
* @brief lp_build_fetch_rgba_aos_array
*
* \param format_desc describes format of the image we're fetching from
* \param dst_type output type
* \param base_ptr address of the pixel block (or the texel if uncompressed)
* \param offset ptr offset
*/
LLVMValueRef
lp_build_fetch_rgba_aos_array(struct gallivm_state *gallivm,
const struct util_format_description *format_desc,
struct lp_type dst_type,
LLVMValueRef base_ptr,
LLVMValueRef offset)
{
struct lp_build_context bld;
LLVMBuilderRef builder = gallivm->builder;
LLVMTypeRef src_vec_type;
LLVMValueRef ptr, res = NULL;
struct lp_type src_type;
boolean pure_integer = format_desc->channel[0].pure_integer;
struct lp_type tmp_type;
lp_type_from_format_desc(&src_type, format_desc);
assert(src_type.length <= dst_type.length);
src_vec_type = lp_build_vec_type(gallivm, src_type);
/* Read whole vector from memory, unaligned */
ptr = LLVMBuildGEP(builder, base_ptr, &offset, 1, "");
ptr = LLVMBuildPointerCast(builder, ptr, LLVMPointerType(src_vec_type, 0), "");
res = LLVMBuildLoad(builder, ptr, "");
LLVMSetAlignment(res, src_type.width / 8);
/* Truncate doubles to float */
if (src_type.floating && src_type.width == 64) {
src_type.width = 32;
src_vec_type = lp_build_vec_type(gallivm, src_type);
res = LLVMBuildFPTrunc(builder, res, src_vec_type, "");
}
/* Expand to correct length */
if (src_type.length < dst_type.length) {
res = lp_build_pad_vector(gallivm, res, dst_type.length);
src_type.length = dst_type.length;
}
tmp_type = dst_type;
if (pure_integer) {
/* some callers expect (fake) floats other real ints. */
tmp_type.floating = 0;
tmp_type.sign = src_type.sign;
}
/* Convert to correct format */
lp_build_conv(gallivm, src_type, tmp_type, &res, 1, &res, 1);
/* Swizzle it */
lp_build_context_init(&bld, gallivm, tmp_type);
res = lp_build_format_swizzle_aos(format_desc, &bld, res);
/* Bitcast to floats (for pure integers) when requested */
if (pure_integer && dst_type.floating) {
res = LLVMBuildBitCast(builder, res, lp_build_vec_type(gallivm, dst_type), "");
}
return res;
}
