static void emit_umod(const struct lp_build_tgsi_action *action,
struct lp_build_tgsi_context *bld_base,
struct lp_build_emit_data *emit_data)
{
LLVMBuilderRef builder = bld_base->base.gallivm->builder;
emit_data->output[emit_data->chan] = LLVMBuildURem(builder,
emit_data->args[0], emit_data->args[1], "");
}
/**
* Build LLVM code for texture coord wrapping, for nearest filtering,
* for scaled integer texcoords.
* \param block_length is the length of the pixel block along the
* coordinate axis
* \param coord the incoming texcoord (s,t,r or q) scaled to the texture size
* \param length the texture size along one dimension
* \param stride pixel stride along the coordinate axis (in bytes)
* \param is_pot if TRUE, length is a power of two
* \param wrap_mode one of PIPE_TEX_WRAP_x
* \param out_offset byte offset for the wrapped coordinate
* \param out_i resulting sub-block pixel coordinate for coord0
*/
static void
lp_build_sample_wrap_nearest_int(struct lp_build_sample_context *bld,
unsigned block_length,
LLVMValueRef coord,
LLVMValueRef length,
LLVMValueRef stride,
boolean is_pot,
unsigned wrap_mode,
LLVMValueRef *out_offset,
LLVMValueRef *out_i)
{
struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
LLVMBuilderRef builder = bld->gallivm->builder;
LLVMValueRef length_minus_one;
length_minus_one = lp_build_sub(int_coord_bld, length, int_coord_bld->one);
switch(wrap_mode) {
case PIPE_TEX_WRAP_REPEAT:
if(is_pot)
coord = LLVMBuildAnd(builder, coord, length_minus_one, "");
else {
/* Add a bias to the texcoord to handle negative coords */
LLVMValueRef bias = lp_build_mul_imm(int_coord_bld, length, 1024);
coord = LLVMBuildAdd(builder, coord, bias, "");
coord = LLVMBuildURem(builder, coord, length, "");
}
break;
case PIPE_TEX_WRAP_CLAMP_TO_EDGE:
coord = lp_build_max(int_coord_bld, coord, int_coord_bld->zero);
coord = lp_build_min(int_coord_bld, coord, length_minus_one);
break;
case PIPE_TEX_WRAP_CLAMP:
case PIPE_TEX_WRAP_CLAMP_TO_BORDER:
case PIPE_TEX_WRAP_MIRROR_REPEAT:
case PIPE_TEX_WRAP_MIRROR_CLAMP:
case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE:
case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER:
default:
assert(0);
}
lp_build_sample_partial_offset(int_coord_bld, block_length, coord, stride,
out_offset, out_i);
}
/**
* Compute the partial offset of a pixel block along an arbitrary axis.
*
* @param coord coordinate in pixels
* @param stride number of bytes between rows of successive pixel blocks
* @param block_length number of pixels in a pixels block along the coordinate
* axis
* @param out_offset resulting relative offset of the pixel block in bytes
* @param out_subcoord resulting sub-block pixel coordinate
*/
void
lp_build_sample_partial_offset(struct lp_build_context *bld,
unsigned block_length,
LLVMValueRef coord,
LLVMValueRef stride,
LLVMValueRef *out_offset,
LLVMValueRef *out_subcoord)
{
LLVMBuilderRef builder = bld->gallivm->builder;
LLVMValueRef offset;
LLVMValueRef subcoord;
if (block_length == 1) {
subcoord = bld->zero;
}
else {
/*
* Pixel blocks have power of two dimensions. LLVM should convert the
* rem/div to bit arithmetic.
* TODO: Verify this.
* It does indeed BUT it does transform it to scalar (and back) when doing so
* (using roughly extract, shift/and, mov, unpack) (llvm 2.7).
* The generated code looks seriously unfunny and is quite expensive.
*/
#if 0
LLVMValueRef block_width = lp_build_const_int_vec(bld->type, block_length);
subcoord = LLVMBuildURem(builder, coord, block_width, "");
coord = LLVMBuildUDiv(builder, coord, block_width, "");
#else
unsigned logbase2 = util_logbase2(block_length);
LLVMValueRef block_shift = lp_build_const_int_vec(bld->gallivm, bld->type, logbase2);
LLVMValueRef block_mask = lp_build_const_int_vec(bld->gallivm, bld->type, block_length - 1);
subcoord = LLVMBuildAnd(builder, coord, block_mask, "");
coord = LLVMBuildLShr(builder, coord, block_shift, "");
#endif
}
offset = lp_build_mul(bld, coord, stride);
assert(out_offset);
assert(out_subcoord);
*out_offset = offset;
*out_subcoord = subcoord;
}
/*
* gen_operator_expression
*
* Code generation for operator expressions. Most of them have straightforward
* translations into LLVM instructions and are handled directly here.
*/
static LLVMValueRef
gen_operator_expression (gencodectx_t gctx, expr_node_t *exp, LLVMTypeRef neededtype)
{
expr_node_t *lhs = expr_op_lhs(exp);
expr_node_t *rhs = expr_op_rhs(exp);
optype_t op = expr_op_type(exp);
LLVMBuilderRef builder = gctx->curfn->builder;
LLVMTypeRef inttype;
LLVMValueRef lval, rval, result;
if (op == OPER_FETCH) {
return gen_fetch(gctx, rhs, neededtype);
}
if (op == OPER_ASSIGN) {
LLVMValueRef val = llvmgen_assignment(gctx, lhs, rhs);
return llvmgen_adjustval(gctx, val, neededtype, 0);
}
if (op == OPER_SHIFT) {
return gen_shift(gctx, lhs, rhs, neededtype);
}
inttype = LLVMIntTypeInContext(gctx->llvmctx, machine_scalar_bits(gctx->mach));
lval = (lhs == 0 ? 0 : llvmgen_expression(gctx, lhs, inttype));
rval = llvmgen_expression(gctx, rhs, inttype);
switch (op) {
case OPER_UNARY_PLUS:
result = rval;
break;
case OPER_UNARY_MINUS:
result = LLVMBuildNeg(builder, rval, llvmgen_temp(gctx));
break;
case OPER_ADD:
result = LLVMBuildAdd(builder, lval, rval, llvmgen_temp(gctx));
break;
case OPER_SUBTRACT:
result = LLVMBuildSub(builder, lval, rval, llvmgen_temp(gctx));
break;
case OPER_MULT:
result = LLVMBuildMul(builder, lval, rval, llvmgen_temp(gctx));
break;
case OPER_DIV:
result = LLVMBuildUDiv(builder, lval, rval, llvmgen_temp(gctx));
break;
case OPER_MODULO:
result = LLVMBuildURem(builder, lval, rval, llvmgen_temp(gctx));
break;
case OPER_AND:
result = LLVMBuildAnd(builder, lval, rval, llvmgen_temp(gctx));
break;
case OPER_OR:
result = LLVMBuildOr(builder, lval, rval, llvmgen_temp(gctx));
break;
case OPER_NOT:
result = LLVMBuildNot(builder, rval, llvmgen_temp(gctx));
break;
case OPER_XOR:
result = LLVMBuildXor(builder, lval, rval, llvmgen_temp(gctx));
break;
case OPER_EQV:
result = LLVMBuildXor(builder, lval, rval, llvmgen_temp(gctx));
result = LLVMBuildNot(builder, result, llvmgen_temp(gctx));
break;
default:
if (op >= OPER_CMP_EQL && op <= OPER_CMP_GEQA) {
result = LLVMBuildICmp(builder,
llvmgen_predfromop(op, machine_addr_signed(gctx->mach)),
lval, rval, llvmgen_temp(gctx));
} else {
// Everything should be covered
expr_signal(gctx->ectx, STC__INTCMPERR, "gen_operator_expression");
result = LLVMConstNull(inttype);
}
break;
}
return llvmgen_adjustval(gctx, result, neededtype, 0);
} /* gen_operator_expression */
/**
* Build LLVM code for texture coord wrapping, for linear filtering,
* for scaled integer texcoords.
* \param block_length is the length of the pixel block along the
* coordinate axis
* \param coord0 the incoming texcoord (s,t,r or q) scaled to the texture size
* \param length the texture size along one dimension
* \param stride pixel stride along the coordinate axis (in bytes)
* \param is_pot if TRUE, length is a power of two
* \param wrap_mode one of PIPE_TEX_WRAP_x
* \param offset0 resulting relative offset for coord0
* \param offset1 resulting relative offset for coord0 + 1
* \param i0 resulting sub-block pixel coordinate for coord0
* \param i1 resulting sub-block pixel coordinate for coord0 + 1
*/
static void
lp_build_sample_wrap_linear_int(struct lp_build_sample_context *bld,
unsigned block_length,
LLVMValueRef coord0,
LLVMValueRef length,
LLVMValueRef stride,
boolean is_pot,
unsigned wrap_mode,
LLVMValueRef *offset0,
LLVMValueRef *offset1,
LLVMValueRef *i0,
LLVMValueRef *i1)
{
struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
LLVMBuilderRef builder = bld->gallivm->builder;
LLVMValueRef length_minus_one;
LLVMValueRef lmask, umask, mask;
if (block_length != 1) {
/*
* If the pixel block covers more than one pixel then there is no easy
* way to calculate offset1 relative to offset0. Instead, compute them
* independently.
*/
LLVMValueRef coord1;
lp_build_sample_wrap_nearest_int(bld,
block_length,
coord0,
length,
stride,
is_pot,
wrap_mode,
offset0, i0);
coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
lp_build_sample_wrap_nearest_int(bld,
block_length,
coord1,
length,
stride,
is_pot,
wrap_mode,
offset1, i1);
return;
}
/*
* Scalar pixels -- try to compute offset0 and offset1 with a single stride
* multiplication.
*/
*i0 = int_coord_bld->zero;
*i1 = int_coord_bld->zero;
length_minus_one = lp_build_sub(int_coord_bld, length, int_coord_bld->one);
switch(wrap_mode) {
case PIPE_TEX_WRAP_REPEAT:
if (is_pot) {
coord0 = LLVMBuildAnd(builder, coord0, length_minus_one, "");
}
else {
/* Add a bias to the texcoord to handle negative coords */
LLVMValueRef bias = lp_build_mul_imm(int_coord_bld, length, 1024);
coord0 = LLVMBuildAdd(builder, coord0, bias, "");
coord0 = LLVMBuildURem(builder, coord0, length, "");
}
mask = lp_build_compare(bld->gallivm, int_coord_bld->type,
PIPE_FUNC_NOTEQUAL, coord0, length_minus_one);
*offset0 = lp_build_mul(int_coord_bld, coord0, stride);
*offset1 = LLVMBuildAnd(builder,
lp_build_add(int_coord_bld, *offset0, stride),
mask, "");
break;
case PIPE_TEX_WRAP_CLAMP_TO_EDGE:
lmask = lp_build_compare(int_coord_bld->gallivm, int_coord_bld->type,
PIPE_FUNC_GEQUAL, coord0, int_coord_bld->zero);
umask = lp_build_compare(int_coord_bld->gallivm, int_coord_bld->type,
PIPE_FUNC_LESS, coord0, length_minus_one);
coord0 = lp_build_select(int_coord_bld, lmask, coord0, int_coord_bld->zero);
coord0 = lp_build_select(int_coord_bld, umask, coord0, length_minus_one);
mask = LLVMBuildAnd(builder, lmask, umask, "");
*offset0 = lp_build_mul(int_coord_bld, coord0, stride);
*offset1 = lp_build_add(int_coord_bld,
*offset0,
LLVMBuildAnd(builder, stride, mask, ""));
break;
case PIPE_TEX_WRAP_CLAMP:
case PIPE_TEX_WRAP_CLAMP_TO_BORDER:
case PIPE_TEX_WRAP_MIRROR_REPEAT:
case PIPE_TEX_WRAP_MIRROR_CLAMP:
case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE:
case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER:
default:
assert(0);
*offset0 = int_coord_bld->zero;
*offset1 = int_coord_bld->zero;
break;
}
}
void JITImpl::
emitMemoryChecks(unsigned index,
std::queue<std::pair<uint32_t,MemoryCheck*> > &checks)
{
while (!checks.empty() && checks.front().first == index) {
MemoryCheck *check = checks.front().second;
checks.pop();
LLVMBasicBlockRef bailoutBB = getOrCreateMemoryCheckBailoutBlock(index);
// Compute address.
LLVMValueRef address;
{
LLVMTypeRef paramTypes[5];
LLVMGetParamTypes(LLVMGetElementType(LLVMTypeOf(functions.jitComputeAddress)),
paramTypes);
LLVMValueRef args[] = {
threadParam,
LLVMConstInt(paramTypes[1], check->getBaseReg(), false),
LLVMConstInt(paramTypes[2], check->getScale(), false),
LLVMConstInt(paramTypes[3], check->getOffsetReg(), false),
LLVMConstInt(paramTypes[4], check->getOffsetImm(), false)
};
address = emitCallToBeInlined(functions.jitComputeAddress, args, 5);
}
// Check alignment.
if (check->getFlags() & MemoryCheck::CheckAlignment &&
check->getSize() > 1) {
LLVMValueRef rem =
LLVMBuildURem(
builder, address,
LLVMConstInt(LLVMTypeOf(address), check->getSize(), false), "");
LLVMValueRef cmp =
LLVMBuildICmp(builder, LLVMIntNE, rem,
LLVMConstInt(LLVMTypeOf(address), 0, false), "");
emitCondBrToBlock(cmp, bailoutBB);
}
// Check address valid.
if (check->getFlags() & MemoryCheck::CheckAddress) {
LLVMValueRef args[] = {
threadParam,
ramSizeLog2Param,
address
};
LLVMValueRef isValid = emitCallToBeInlined(functions.jitCheckAddress,
args, 3);
LLVMValueRef cmp =
LLVMBuildICmp(builder, LLVMIntEQ, isValid,
LLVMConstInt(LLVMTypeOf(isValid), 0, false), "");
emitCondBrToBlock(cmp, bailoutBB);
}
// Check invalidation info.
if (check->getFlags() & MemoryCheck::CheckInvalidation) {
LLVMValueRef args[] = {
threadParam,
address
};
LLVMValueRef cacheInvalidated =
emitCallToBeInlined(getJitInvalidateFunction(check->getSize()), args,
2);
LLVMValueRef cmp =
LLVMBuildICmp(builder, LLVMIntNE, cacheInvalidated,
LLVMConstInt(LLVMTypeOf(cacheInvalidated), 0, false), "");
emitCondBrToBlock(cmp, bailoutBB);
}
delete check;
}
}
