/**
* Bit mask conversion.
*
* This will convert the integer masks that match the given types.
*
* The mask values should 0 or -1, i.e., all bits either set to zero or one.
* Any other value will likely cause in unpredictable results.
*
* This is basically a very trimmed down version of lp_build_conv.
*/
void
lp_build_conv_mask(LLVMBuilderRef builder,
struct lp_type src_type,
struct lp_type dst_type,
const LLVMValueRef *src, unsigned num_srcs,
LLVMValueRef *dst, unsigned num_dsts)
{
/* Register width must remain constant */
assert(src_type.width * src_type.length == dst_type.width * dst_type.length);
/* We must not loose or gain channels. Only precision */
assert(src_type.length * num_srcs == dst_type.length * num_dsts);
/*
* Drop
*
* We assume all values are 0 or -1
*/
src_type.floating = FALSE;
src_type.fixed = FALSE;
src_type.sign = TRUE;
src_type.norm = FALSE;
dst_type.floating = FALSE;
dst_type.fixed = FALSE;
dst_type.sign = TRUE;
dst_type.norm = FALSE;
/*
* Truncate or expand bit width
*/
if(src_type.width > dst_type.width) {
assert(num_dsts == 1);
dst[0] = lp_build_pack(builder, src_type, dst_type, TRUE, src, num_srcs);
}
else if(src_type.width < dst_type.width) {
assert(num_srcs == 1);
lp_build_unpack(builder, src_type, dst_type, src[0], dst, num_dsts);
}
else {
assert(num_srcs == num_dsts);
memcpy(dst, src, num_dsts * sizeof *dst);
}
}
/**
* Truncate or expand the bitwidth.
*
* NOTE: Getting the right sign flags is crucial here, as we employ some
* intrinsics that do saturation.
*/
void
lp_build_resize(struct gallivm_state *gallivm,
struct lp_type src_type,
struct lp_type dst_type,
const LLVMValueRef *src, unsigned num_srcs,
LLVMValueRef *dst, unsigned num_dsts)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH];
unsigned i;
/*
* We don't support float <-> int conversion here. That must be done
* before/after calling this function.
*/
assert(src_type.floating == dst_type.floating);
/*
* We don't support double <-> float conversion yet, although it could be
* added with little effort.
*/
assert((!src_type.floating && !dst_type.floating) ||
src_type.width == dst_type.width);
/* We must not loose or gain channels. Only precision */
assert(src_type.length * num_srcs == dst_type.length * num_dsts);
/* We don't support M:N conversion, only 1:N, M:1, or 1:1 */
assert(num_srcs == 1 || num_dsts == 1);
assert(src_type.length <= LP_MAX_VECTOR_LENGTH);
assert(dst_type.length <= LP_MAX_VECTOR_LENGTH);
assert(num_srcs <= LP_MAX_VECTOR_LENGTH);
assert(num_dsts <= LP_MAX_VECTOR_LENGTH);
if (src_type.width > dst_type.width) {
/*
* Truncate bit width.
*/
assert(num_dsts == 1);
if (src_type.width * src_type.length == dst_type.width * dst_type.length) {
/*
* Register width remains constant -- use vector packing intrinsics
*/
tmp[0] = lp_build_pack(gallivm, src_type, dst_type, TRUE, src, num_srcs);
}
else {
if (src_type.width / dst_type.width > num_srcs) {
/*
* First change src vectors size (with shuffle) so they have the
* same size as the destination vector, then pack normally.
* Note: cannot use cast/extract because llvm generates atrocious code.
*/
unsigned size_ratio = (src_type.width * src_type.length) /
(dst_type.length * dst_type.width);
unsigned new_length = src_type.length / size_ratio;
for (i = 0; i < size_ratio * num_srcs; i++) {
unsigned start_index = (i % size_ratio) * new_length;
tmp[i] = lp_build_extract_range(gallivm, src[i / size_ratio],
start_index, new_length);
}
num_srcs *= size_ratio;
src_type.length = new_length;
tmp[0] = lp_build_pack(gallivm, src_type, dst_type, TRUE, tmp, num_srcs);
}
else {
/*
* Truncate bit width but expand vector size - first pack
* then expand simply because this should be more AVX-friendly
* for the cases we probably hit.
*/
unsigned size_ratio = (dst_type.width * dst_type.length) /
(src_type.length * src_type.width);
unsigned num_pack_srcs = num_srcs / size_ratio;
dst_type.length = dst_type.length / size_ratio;
for (i = 0; i < size_ratio; i++) {
tmp[i] = lp_build_pack(gallivm, src_type, dst_type, TRUE,
&src[i*num_pack_srcs], num_pack_srcs);
}
tmp[0] = lp_build_concat(gallivm, tmp, dst_type, size_ratio);
}
}
}
else if (src_type.width < dst_type.width) {
/*
* Expand bit width.
*/
assert(num_srcs == 1);
if (src_type.width * src_type.length == dst_type.width * dst_type.length) {
/*
* Register width remains constant -- use vector unpack intrinsics
*/
lp_build_unpack(gallivm, src_type, dst_type, src[0], tmp, num_dsts);
}
else {
/*
* Do it element-wise.
*/
assert(src_type.length * num_srcs == dst_type.length * num_dsts);
for (i = 0; i < num_dsts; i++) {
tmp[i] = lp_build_undef(gallivm, dst_type);
}
for (i = 0; i < src_type.length; ++i) {
unsigned j = i / dst_type.length;
LLVMValueRef srcindex = lp_build_const_int32(gallivm, i);
LLVMValueRef dstindex = lp_build_const_int32(gallivm, i % dst_type.length);
LLVMValueRef val = LLVMBuildExtractElement(builder, src[0], srcindex, "");
if (src_type.sign && dst_type.sign) {
val = LLVMBuildSExt(builder, val, lp_build_elem_type(gallivm, dst_type), "");
} else {
val = LLVMBuildZExt(builder, val, lp_build_elem_type(gallivm, dst_type), "");
}
tmp[j] = LLVMBuildInsertElement(builder, tmp[j], val, dstindex, "");
}
}
}
else {
/*
* No-op
*/
assert(num_srcs == 1);
assert(num_dsts == 1);
tmp[0] = src[0];
}
for(i = 0; i < num_dsts; ++i)
dst[i] = tmp[i];
}
/**
* Generic type conversion.
*
* TODO: Take a precision argument, or even better, add a new precision member
* to the lp_type union.
*/
void
lp_build_conv(LLVMBuilderRef builder,
struct lp_type src_type,
struct lp_type dst_type,
const LLVMValueRef *src, unsigned num_srcs,
LLVMValueRef *dst, unsigned num_dsts)
{
struct lp_type tmp_type;
LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH];
unsigned num_tmps;
unsigned i;
/* Register width must remain constant */
assert(src_type.width * src_type.length == dst_type.width * dst_type.length);
/* We must not loose or gain channels. Only precision */
assert(src_type.length * num_srcs == dst_type.length * num_dsts);
assert(src_type.length <= LP_MAX_VECTOR_LENGTH);
assert(dst_type.length <= LP_MAX_VECTOR_LENGTH);
tmp_type = src_type;
for(i = 0; i < num_srcs; ++i)
tmp[i] = src[i];
num_tmps = num_srcs;
/*
* Clamp if necessary
*/
if(memcmp(&src_type, &dst_type, sizeof src_type) != 0) {
struct lp_build_context bld;
double src_min = lp_const_min(src_type);
double dst_min = lp_const_min(dst_type);
double src_max = lp_const_max(src_type);
double dst_max = lp_const_max(dst_type);
LLVMValueRef thres;
lp_build_context_init(&bld, builder, tmp_type);
if(src_min < dst_min) {
if(dst_min == 0.0)
thres = bld.zero;
else
thres = lp_build_const_scalar(src_type, dst_min);
for(i = 0; i < num_tmps; ++i)
tmp[i] = lp_build_max(&bld, tmp[i], thres);
}
if(src_max > dst_max) {
if(dst_max == 1.0)
thres = bld.one;
else
thres = lp_build_const_scalar(src_type, dst_max);
for(i = 0; i < num_tmps; ++i)
tmp[i] = lp_build_min(&bld, tmp[i], thres);
}
}
/*
* Scale to the narrowest range
*/
if(dst_type.floating) {
/* Nothing to do */
}
else if(tmp_type.floating) {
if(!dst_type.fixed && !dst_type.sign && dst_type.norm) {
for(i = 0; i < num_tmps; ++i) {
tmp[i] = lp_build_clamped_float_to_unsigned_norm(builder,
tmp_type,
dst_type.width,
tmp[i]);
}
tmp_type.floating = FALSE;
}
else {
double dst_scale = lp_const_scale(dst_type);
LLVMTypeRef tmp_vec_type;
if (dst_scale != 1.0) {
LLVMValueRef scale = lp_build_const_scalar(tmp_type, dst_scale);
for(i = 0; i < num_tmps; ++i)
tmp[i] = LLVMBuildMul(builder, tmp[i], scale, "");
}
/* Use an equally sized integer for intermediate computations */
tmp_type.floating = FALSE;
tmp_vec_type = lp_build_vec_type(tmp_type);
for(i = 0; i < num_tmps; ++i) {
#if 0
if(dst_type.sign)
tmp[i] = LLVMBuildFPToSI(builder, tmp[i], tmp_vec_type, "");
else
tmp[i] = LLVMBuildFPToUI(builder, tmp[i], tmp_vec_type, "");
#else
/* FIXME: there is no SSE counterpart for LLVMBuildFPToUI */
tmp[i] = LLVMBuildFPToSI(builder, tmp[i], tmp_vec_type, "");
#endif
}
}
}
else {
unsigned src_shift = lp_const_shift(src_type);
unsigned dst_shift = lp_const_shift(dst_type);
/* FIXME: compensate different offsets too */
if(src_shift > dst_shift) {
LLVMValueRef shift = lp_build_int_const_scalar(tmp_type, src_shift - dst_shift);
for(i = 0; i < num_tmps; ++i)
if(src_type.sign)
tmp[i] = LLVMBuildAShr(builder, tmp[i], shift, "");
else
tmp[i] = LLVMBuildLShr(builder, tmp[i], shift, "");
}
}
/*
* Truncate or expand bit width
*/
assert(!tmp_type.floating || tmp_type.width == dst_type.width);
if(tmp_type.width > dst_type.width) {
assert(num_dsts == 1);
tmp[0] = lp_build_pack(builder, tmp_type, dst_type, TRUE, tmp, num_tmps);
tmp_type.width = dst_type.width;
tmp_type.length = dst_type.length;
num_tmps = 1;
}
if(tmp_type.width < dst_type.width) {
assert(num_tmps == 1);
lp_build_unpack(builder, tmp_type, dst_type, tmp[0], tmp, num_dsts);
tmp_type.width = dst_type.width;
tmp_type.length = dst_type.length;
num_tmps = num_dsts;
}
assert(tmp_type.width == dst_type.width);
assert(tmp_type.length == dst_type.length);
assert(num_tmps == num_dsts);
/*
* Scale to the widest range
*/
if(src_type.floating) {
/* Nothing to do */
}
else if(!src_type.floating && dst_type.floating) {
if(!src_type.fixed && !src_type.sign && src_type.norm) {
for(i = 0; i < num_tmps; ++i) {
tmp[i] = lp_build_unsigned_norm_to_float(builder,
src_type.width,
dst_type,
tmp[i]);
}
tmp_type.floating = TRUE;
}
else {
double src_scale = lp_const_scale(src_type);
LLVMTypeRef tmp_vec_type;
/* Use an equally sized integer for intermediate computations */
tmp_type.floating = TRUE;
tmp_type.sign = TRUE;
tmp_vec_type = lp_build_vec_type(tmp_type);
for(i = 0; i < num_tmps; ++i) {
#if 0
if(dst_type.sign)
tmp[i] = LLVMBuildSIToFP(builder, tmp[i], tmp_vec_type, "");
else
tmp[i] = LLVMBuildUIToFP(builder, tmp[i], tmp_vec_type, "");
#else
/* FIXME: there is no SSE counterpart for LLVMBuildUIToFP */
tmp[i] = LLVMBuildSIToFP(builder, tmp[i], tmp_vec_type, "");
#endif
}
if (src_scale != 1.0) {
LLVMValueRef scale = lp_build_const_scalar(tmp_type, 1.0/src_scale);
for(i = 0; i < num_tmps; ++i)
tmp[i] = LLVMBuildMul(builder, tmp[i], scale, "");
}
}
}
else {
unsigned src_shift = lp_const_shift(src_type);
unsigned dst_shift = lp_const_shift(dst_type);
/* FIXME: compensate different offsets too */
if(src_shift < dst_shift) {
LLVMValueRef shift = lp_build_int_const_scalar(tmp_type, dst_shift - src_shift);
for(i = 0; i < num_tmps; ++i)
tmp[i] = LLVMBuildShl(builder, tmp[i], shift, "");
}
}
for(i = 0; i < num_dsts; ++i)
dst[i] = tmp[i];
}