/**
* Special case for converting clamped IEEE-754 floats to unsigned norms.
*
* The mathematical voodoo below may seem excessive but it is actually
* paramount we do it this way for several reasons. First, there is no single
* precision FP to unsigned integer conversion Intel SSE instruction. Second,
* secondly, even if there was, since the FP's mantissa takes only a fraction
* of register bits the typically scale and cast approach would require double
* precision for accurate results, and therefore half the throughput
*
* Although the result values can be scaled to an arbitrary bit width specified
* by dst_width, the actual result type will have the same width.
*/
LLVMValueRef
lp_build_clamped_float_to_unsigned_norm(LLVMBuilderRef builder,
struct lp_type src_type,
unsigned dst_width,
LLVMValueRef src)
{
LLVMTypeRef int_vec_type = lp_build_int_vec_type(src_type);
LLVMValueRef res;
unsigned mantissa;
unsigned n;
unsigned long long ubound;
unsigned long long mask;
double scale;
double bias;
assert(src_type.floating);
mantissa = lp_mantissa(src_type);
/* We cannot carry more bits than the mantissa */
n = MIN2(mantissa, dst_width);
/* This magic coefficients will make the desired result to appear in the
* lowest significant bits of the mantissa.
*/
ubound = ((unsigned long long)1 << n);
mask = ubound - 1;
scale = (double)mask/ubound;
bias = (double)((unsigned long long)1 << (mantissa - n));
res = LLVMBuildMul(builder, src, lp_build_const_scalar(src_type, scale), "");
res = LLVMBuildAdd(builder, res, lp_build_const_scalar(src_type, bias), "");
res = LLVMBuildBitCast(builder, res, int_vec_type, "");
if(dst_width > n) {
int shift = dst_width - n;
res = LLVMBuildShl(builder, res, lp_build_int_const_scalar(src_type, shift), "");
/* TODO: Fill in the empty lower bits for additional precision? */
#if 0
{
LLVMValueRef msb;
msb = LLVMBuildLShr(builder, res, lp_build_int_const_scalar(src_type, dst_width - 1), "");
msb = LLVMBuildShl(builder, msb, lp_build_int_const_scalar(src_type, shift), "");
msb = LLVMBuildSub(builder, msb, lp_build_int_const_scalar(src_type, 1), "");
res = LLVMBuildOr(builder, res, msb, "");
}
#elif 0
while(shift > 0) {
res = LLVMBuildOr(builder, res, LLVMBuildLShr(builder, res, lp_build_int_const_scalar(src_type, n), ""), "");
shift -= n;
n *= 2;
}
#endif
}
else
res = LLVMBuildAnd(builder, res, lp_build_int_const_scalar(src_type, mask), "");
return res;
}
void
lp_build_exp2_approx(struct lp_build_context *bld,
LLVMValueRef x,
LLVMValueRef *p_exp2_int_part,
LLVMValueRef *p_frac_part,
LLVMValueRef *p_exp2)
{
const struct lp_type type = bld->type;
LLVMTypeRef vec_type = lp_build_vec_type(type);
LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
LLVMValueRef ipart = NULL;
LLVMValueRef fpart = NULL;
LLVMValueRef expipart = NULL;
LLVMValueRef expfpart = NULL;
LLVMValueRef res = NULL;
if(p_exp2_int_part || p_frac_part || p_exp2) {
/* TODO: optimize the constant case */
if(LLVMIsConstant(x))
debug_printf("%s: inefficient/imprecise constant arithmetic\n",
__FUNCTION__);
assert(type.floating && type.width == 32);
x = lp_build_min(bld, x, lp_build_const_scalar(type, 129.0));
x = lp_build_max(bld, x, lp_build_const_scalar(type, -126.99999));
/* ipart = int(x - 0.5) */
ipart = LLVMBuildSub(bld->builder, x, lp_build_const_scalar(type, 0.5f), "");
ipart = LLVMBuildFPToSI(bld->builder, ipart, int_vec_type, "");
/* fpart = x - ipart */
fpart = LLVMBuildSIToFP(bld->builder, ipart, vec_type, "");
fpart = LLVMBuildSub(bld->builder, x, fpart, "");
}
if(p_exp2_int_part || p_exp2) {
/* expipart = (float) (1 << ipart) */
expipart = LLVMBuildAdd(bld->builder, ipart, lp_build_int_const_scalar(type, 127), "");
expipart = LLVMBuildShl(bld->builder, expipart, lp_build_int_const_scalar(type, 23), "");
expipart = LLVMBuildBitCast(bld->builder, expipart, vec_type, "");
}
if(p_exp2) {
expfpart = lp_build_polynomial(bld, fpart, lp_build_exp2_polynomial,
Elements(lp_build_exp2_polynomial));
res = LLVMBuildMul(bld->builder, expipart, expfpart, "");
}
if(p_exp2_int_part)
*p_exp2_int_part = expipart;
if(p_frac_part)
*p_frac_part = fpart;
if(p_exp2)
*p_exp2 = res;
}
/**
* Compute the offset of a pixel.
*
* x, y, y_stride are vectors
*/
LLVMValueRef
lp_build_sample_offset(struct lp_build_context *bld,
const struct util_format_description *format_desc,
LLVMValueRef x,
LLVMValueRef y,
LLVMValueRef y_stride,
LLVMValueRef data_ptr)
{
LLVMValueRef x_stride;
LLVMValueRef offset;
x_stride = lp_build_const_scalar(bld->type, format_desc->block.bits/8);
if(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS) {
LLVMValueRef x_lo, x_hi;
LLVMValueRef y_lo, y_hi;
LLVMValueRef x_stride_lo, x_stride_hi;
LLVMValueRef y_stride_lo, y_stride_hi;
LLVMValueRef x_offset_lo, x_offset_hi;
LLVMValueRef y_offset_lo, y_offset_hi;
LLVMValueRef offset_lo, offset_hi;
x_lo = LLVMBuildAnd(bld->builder, x, bld->one, "");
y_lo = LLVMBuildAnd(bld->builder, y, bld->one, "");
x_hi = LLVMBuildLShr(bld->builder, x, bld->one, "");
y_hi = LLVMBuildLShr(bld->builder, y, bld->one, "");
x_stride_lo = x_stride;
y_stride_lo = lp_build_const_scalar(bld->type, 2*format_desc->block.bits/8);
x_stride_hi = lp_build_const_scalar(bld->type, 4*format_desc->block.bits/8);
y_stride_hi = LLVMBuildShl(bld->builder, y_stride, bld->one, "");
x_offset_lo = lp_build_mul(bld, x_lo, x_stride_lo);
y_offset_lo = lp_build_mul(bld, y_lo, y_stride_lo);
offset_lo = lp_build_add(bld, x_offset_lo, y_offset_lo);
x_offset_hi = lp_build_mul(bld, x_hi, x_stride_hi);
y_offset_hi = lp_build_mul(bld, y_hi, y_stride_hi);
offset_hi = lp_build_add(bld, x_offset_hi, y_offset_hi);
offset = lp_build_add(bld, offset_hi, offset_lo);
}
else {
LLVMValueRef x_offset;
LLVMValueRef y_offset;
x_offset = lp_build_mul(bld, x, x_stride);
y_offset = lp_build_mul(bld, y, y_stride);
offset = lp_build_add(bld, x_offset, y_offset);
}
return offset;
}
/**
* Small vector x scale multiplication optimization.
*/
LLVMValueRef
lp_build_mul_imm(struct lp_build_context *bld,
LLVMValueRef a,
int b)
{
LLVMValueRef factor;
if(b == 0)
return bld->zero;
if(b == 1)
return a;
if(b == -1)
return LLVMBuildNeg(bld->builder, a, "");
if(b == 2 && bld->type.floating)
return lp_build_add(bld, a, a);
if(util_is_pot(b)) {
unsigned shift = ffs(b) - 1;
if(bld->type.floating) {
#if 0
/*
* Power of two multiplication by directly manipulating the mantissa.
*
* XXX: This might not be always faster, it will introduce a small error
* for multiplication by zero, and it will produce wrong results
* for Inf and NaN.
*/
unsigned mantissa = lp_mantissa(bld->type);
factor = lp_build_int_const_scalar(bld->type, (unsigned long long)shift << mantissa);
a = LLVMBuildBitCast(bld->builder, a, lp_build_int_vec_type(bld->type), "");
a = LLVMBuildAdd(bld->builder, a, factor, "");
a = LLVMBuildBitCast(bld->builder, a, lp_build_vec_type(bld->type), "");
return a;
#endif
}
else {
factor = lp_build_const_scalar(bld->type, shift);
return LLVMBuildShl(bld->builder, a, factor, "");
}
}
factor = lp_build_const_scalar(bld->type, (double)b);
return lp_build_mul(bld, a, factor);
}
/**
* Generate polynomial.
* Ex: coeffs[0] + x * coeffs[1] + x^2 * coeffs[2].
*/
static LLVMValueRef
lp_build_polynomial(struct lp_build_context *bld,
LLVMValueRef x,
const double *coeffs,
unsigned num_coeffs)
{
const struct lp_type type = bld->type;
LLVMValueRef res = NULL;
unsigned i;
/* TODO: optimize the constant case */
if(LLVMIsConstant(x))
debug_printf("%s: inefficient/imprecise constant arithmetic\n",
__FUNCTION__);
for (i = num_coeffs; i--; ) {
LLVMValueRef coeff = lp_build_const_scalar(type, coeffs[i]);
if(res)
res = lp_build_add(bld, coeff, lp_build_mul(bld, x, res));
else
res = coeff;
}
if(res)
return res;
else
return bld->undef;
}
/**
* Inverse of lp_build_clamped_float_to_unsigned_norm above.
*/
LLVMValueRef
lp_build_unsigned_norm_to_float(LLVMBuilderRef builder,
unsigned src_width,
struct lp_type dst_type,
LLVMValueRef src)
{
LLVMTypeRef vec_type = lp_build_vec_type(dst_type);
LLVMTypeRef int_vec_type = lp_build_int_vec_type(dst_type);
LLVMValueRef bias_;
LLVMValueRef res;
unsigned mantissa;
unsigned n;
unsigned long long ubound;
unsigned long long mask;
double scale;
double bias;
mantissa = lp_mantissa(dst_type);
n = MIN2(mantissa, src_width);
ubound = ((unsigned long long)1 << n);
mask = ubound - 1;
scale = (double)ubound/mask;
bias = (double)((unsigned long long)1 << (mantissa - n));
res = src;
if(src_width > mantissa) {
int shift = src_width - mantissa;
res = LLVMBuildLShr(builder, res, lp_build_int_const_scalar(dst_type, shift), "");
}
bias_ = lp_build_const_scalar(dst_type, bias);
res = LLVMBuildOr(builder,
res,
LLVMBuildBitCast(builder, bias_, int_vec_type, ""), "");
res = LLVMBuildBitCast(builder, res, vec_type, "");
res = LLVMBuildSub(builder, res, bias_, "");
res = LLVMBuildMul(builder, res, lp_build_const_scalar(dst_type, scale), "");
return res;
}
/**
* Generate log(x)
*/
LLVMValueRef
lp_build_log(struct lp_build_context *bld,
LLVMValueRef x)
{
/* log(2) */
LLVMValueRef log2 = lp_build_const_scalar(bld->type, 0.69314718055994529);
return lp_build_mul(bld, log2, lp_build_exp2(bld, x));
}
/**
* Generate exp(x)
*/
LLVMValueRef
lp_build_exp(struct lp_build_context *bld,
LLVMValueRef x)
{
/* log2(e) = 1/log(2) */
LLVMValueRef log2e = lp_build_const_scalar(bld->type, 1.4426950408889634);
return lp_build_mul(bld, log2e, lp_build_exp2(bld, x));
}
/**
* Convert float[] to int[] with floor().
*/
LLVMValueRef
lp_build_ifloor(struct lp_build_context *bld,
LLVMValueRef a)
{
const struct lp_type type = bld->type;
LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
LLVMValueRef res;
assert(type.floating);
assert(lp_check_value(type, a));
if(util_cpu_caps.has_sse4_1) {
res = lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_FLOOR);
}
else {
/* Take the sign bit and add it to 1 constant */
LLVMTypeRef vec_type = lp_build_vec_type(type);
unsigned mantissa = lp_mantissa(type);
LLVMValueRef mask = lp_build_int_const_scalar(type, (unsigned long long)1 << (type.width - 1));
LLVMValueRef sign;
LLVMValueRef offset;
/* sign = a < 0 ? ~0 : 0 */
sign = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
sign = LLVMBuildAnd(bld->builder, sign, mask, "");
sign = LLVMBuildAShr(bld->builder, sign, lp_build_int_const_scalar(type, type.width - 1), "");
lp_build_name(sign, "floor.sign");
/* offset = -0.99999(9)f */
offset = lp_build_const_scalar(type, -(double)(((unsigned long long)1 << mantissa) - 1)/((unsigned long long)1 << mantissa));
offset = LLVMConstBitCast(offset, int_vec_type);
/* offset = a < 0 ? -0.99999(9)f : 0.0f */
offset = LLVMBuildAnd(bld->builder, offset, sign, "");
offset = LLVMBuildBitCast(bld->builder, offset, vec_type, "");
lp_build_name(offset, "floor.offset");
res = LLVMBuildAdd(bld->builder, a, offset, "");
lp_build_name(res, "floor.res");
}
res = LLVMBuildFPToSI(bld->builder, res, int_vec_type, "");
lp_build_name(res, "floor");
return res;
}
LLVMValueRef
lp_build_sgn(struct lp_build_context *bld,
LLVMValueRef a)
{
const struct lp_type type = bld->type;
LLVMTypeRef vec_type = lp_build_vec_type(type);
LLVMValueRef cond;
LLVMValueRef res;
/* Handle non-zero case */
if(!type.sign) {
/* if not zero then sign must be positive */
res = bld->one;
}
else if(type.floating) {
/* Take the sign bit and add it to 1 constant */
LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
LLVMValueRef mask = lp_build_int_const_scalar(type, (unsigned long long)1 << (type.width - 1));
LLVMValueRef sign;
LLVMValueRef one;
sign = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
sign = LLVMBuildAnd(bld->builder, sign, mask, "");
one = LLVMConstBitCast(bld->one, int_vec_type);
res = LLVMBuildOr(bld->builder, sign, one, "");
res = LLVMBuildBitCast(bld->builder, res, vec_type, "");
}
else
{
LLVMValueRef minus_one = lp_build_const_scalar(type, -1.0);
cond = lp_build_cmp(bld, PIPE_FUNC_GREATER, a, bld->zero);
res = lp_build_select(bld, cond, bld->one, minus_one);
}
/* Handle zero */
cond = lp_build_cmp(bld, PIPE_FUNC_EQUAL, a, bld->zero);
res = lp_build_select(bld, cond, bld->zero, bld->one);
return res;
}
LLVMValueRef
lp_build_iround(struct lp_build_context *bld,
LLVMValueRef a)
{
const struct lp_type type = bld->type;
LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
LLVMValueRef res;
assert(type.floating);
assert(lp_check_value(type, a));
if(util_cpu_caps.has_sse4_1) {
res = lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_NEAREST);
}
else {
LLVMTypeRef vec_type = lp_build_vec_type(type);
LLVMValueRef mask = lp_build_int_const_scalar(type, (unsigned long long)1 << (type.width - 1));
LLVMValueRef sign;
LLVMValueRef half;
/* get sign bit */
sign = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
sign = LLVMBuildAnd(bld->builder, sign, mask, "");
/* sign * 0.5 */
half = lp_build_const_scalar(type, 0.5);
half = LLVMBuildBitCast(bld->builder, half, int_vec_type, "");
half = LLVMBuildOr(bld->builder, sign, half, "");
half = LLVMBuildBitCast(bld->builder, half, vec_type, "");
res = LLVMBuildAdd(bld->builder, a, half, "");
}
res = LLVMBuildFPToSI(bld->builder, res, int_vec_type, "");
return res;
}
/**
* Generic type conversion.
*
* TODO: Take a precision argument, or even better, add a new precision member
* to the lp_type union.
*/
void
lp_build_conv(LLVMBuilderRef builder,
struct lp_type src_type,
struct lp_type dst_type,
const LLVMValueRef *src, unsigned num_srcs,
LLVMValueRef *dst, unsigned num_dsts)
{
struct lp_type tmp_type;
LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH];
unsigned num_tmps;
unsigned i;
/* Register width must remain constant */
assert(src_type.width * src_type.length == dst_type.width * dst_type.length);
/* We must not loose or gain channels. Only precision */
assert(src_type.length * num_srcs == dst_type.length * num_dsts);
assert(src_type.length <= LP_MAX_VECTOR_LENGTH);
assert(dst_type.length <= LP_MAX_VECTOR_LENGTH);
tmp_type = src_type;
for(i = 0; i < num_srcs; ++i)
tmp[i] = src[i];
num_tmps = num_srcs;
/*
* Clamp if necessary
*/
if(memcmp(&src_type, &dst_type, sizeof src_type) != 0) {
struct lp_build_context bld;
double src_min = lp_const_min(src_type);
double dst_min = lp_const_min(dst_type);
double src_max = lp_const_max(src_type);
double dst_max = lp_const_max(dst_type);
LLVMValueRef thres;
lp_build_context_init(&bld, builder, tmp_type);
if(src_min < dst_min) {
if(dst_min == 0.0)
thres = bld.zero;
else
thres = lp_build_const_scalar(src_type, dst_min);
for(i = 0; i < num_tmps; ++i)
tmp[i] = lp_build_max(&bld, tmp[i], thres);
}
if(src_max > dst_max) {
if(dst_max == 1.0)
thres = bld.one;
else
thres = lp_build_const_scalar(src_type, dst_max);
for(i = 0; i < num_tmps; ++i)
tmp[i] = lp_build_min(&bld, tmp[i], thres);
}
}
/*
* Scale to the narrowest range
*/
if(dst_type.floating) {
/* Nothing to do */
}
else if(tmp_type.floating) {
if(!dst_type.fixed && !dst_type.sign && dst_type.norm) {
for(i = 0; i < num_tmps; ++i) {
tmp[i] = lp_build_clamped_float_to_unsigned_norm(builder,
tmp_type,
dst_type.width,
tmp[i]);
}
tmp_type.floating = FALSE;
}
else {
double dst_scale = lp_const_scale(dst_type);
LLVMTypeRef tmp_vec_type;
if (dst_scale != 1.0) {
LLVMValueRef scale = lp_build_const_scalar(tmp_type, dst_scale);
for(i = 0; i < num_tmps; ++i)
tmp[i] = LLVMBuildMul(builder, tmp[i], scale, "");
}
/* Use an equally sized integer for intermediate computations */
tmp_type.floating = FALSE;
tmp_vec_type = lp_build_vec_type(tmp_type);
for(i = 0; i < num_tmps; ++i) {
#if 0
if(dst_type.sign)
tmp[i] = LLVMBuildFPToSI(builder, tmp[i], tmp_vec_type, "");
else
tmp[i] = LLVMBuildFPToUI(builder, tmp[i], tmp_vec_type, "");
#else
/* FIXME: there is no SSE counterpart for LLVMBuildFPToUI */
tmp[i] = LLVMBuildFPToSI(builder, tmp[i], tmp_vec_type, "");
#endif
}
}
}
else {
unsigned src_shift = lp_const_shift(src_type);
unsigned dst_shift = lp_const_shift(dst_type);
/* FIXME: compensate different offsets too */
if(src_shift > dst_shift) {
LLVMValueRef shift = lp_build_int_const_scalar(tmp_type, src_shift - dst_shift);
for(i = 0; i < num_tmps; ++i)
if(src_type.sign)
tmp[i] = LLVMBuildAShr(builder, tmp[i], shift, "");
else
tmp[i] = LLVMBuildLShr(builder, tmp[i], shift, "");
}
}
/*
* Truncate or expand bit width
*/
assert(!tmp_type.floating || tmp_type.width == dst_type.width);
if(tmp_type.width > dst_type.width) {
assert(num_dsts == 1);
tmp[0] = lp_build_pack(builder, tmp_type, dst_type, TRUE, tmp, num_tmps);
tmp_type.width = dst_type.width;
tmp_type.length = dst_type.length;
num_tmps = 1;
}
if(tmp_type.width < dst_type.width) {
assert(num_tmps == 1);
lp_build_unpack(builder, tmp_type, dst_type, tmp[0], tmp, num_dsts);
tmp_type.width = dst_type.width;
tmp_type.length = dst_type.length;
num_tmps = num_dsts;
}
assert(tmp_type.width == dst_type.width);
assert(tmp_type.length == dst_type.length);
assert(num_tmps == num_dsts);
/*
* Scale to the widest range
*/
if(src_type.floating) {
/* Nothing to do */
}
else if(!src_type.floating && dst_type.floating) {
if(!src_type.fixed && !src_type.sign && src_type.norm) {
for(i = 0; i < num_tmps; ++i) {
tmp[i] = lp_build_unsigned_norm_to_float(builder,
src_type.width,
dst_type,
tmp[i]);
}
tmp_type.floating = TRUE;
}
else {
double src_scale = lp_const_scale(src_type);
LLVMTypeRef tmp_vec_type;
/* Use an equally sized integer for intermediate computations */
tmp_type.floating = TRUE;
tmp_type.sign = TRUE;
tmp_vec_type = lp_build_vec_type(tmp_type);
for(i = 0; i < num_tmps; ++i) {
#if 0
if(dst_type.sign)
tmp[i] = LLVMBuildSIToFP(builder, tmp[i], tmp_vec_type, "");
else
tmp[i] = LLVMBuildUIToFP(builder, tmp[i], tmp_vec_type, "");
#else
/* FIXME: there is no SSE counterpart for LLVMBuildUIToFP */
tmp[i] = LLVMBuildSIToFP(builder, tmp[i], tmp_vec_type, "");
#endif
}
if (src_scale != 1.0) {
LLVMValueRef scale = lp_build_const_scalar(tmp_type, 1.0/src_scale);
for(i = 0; i < num_tmps; ++i)
tmp[i] = LLVMBuildMul(builder, tmp[i], scale, "");
}
}
}
else {
unsigned src_shift = lp_const_shift(src_type);
unsigned dst_shift = lp_const_shift(dst_type);
/* FIXME: compensate different offsets too */
if(src_shift < dst_shift) {
LLVMValueRef shift = lp_build_int_const_scalar(tmp_type, dst_shift - src_shift);
for(i = 0; i < num_tmps; ++i)
tmp[i] = LLVMBuildShl(builder, tmp[i], shift, "");
}
}
for(i = 0; i < num_dsts; ++i)
dst[i] = tmp[i];
}
