static LLVMValueRef
emit_fetch_immediate(
struct lp_build_tgsi_context *bld_base,
const struct tgsi_full_src_register *reg,
enum tgsi_opcode_type type,
unsigned swizzle)
{
LLVMTypeRef ctype;
LLVMContextRef ctx = bld_base->base.gallivm->context;
switch (type) {
case TGSI_TYPE_UNSIGNED:
case TGSI_TYPE_SIGNED:
ctype = LLVMInt32TypeInContext(ctx);
break;
case TGSI_TYPE_UNTYPED:
case TGSI_TYPE_FLOAT:
ctype = LLVMFloatTypeInContext(ctx);
break;
default:
ctype = 0;
break;
}
struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
return LLVMConstBitCast(bld->immediates[reg->Register.Index][swizzle], ctype);
}
/* returns a LLVM representation corresponding to the C translation of the
* given IDL type.
*/
LLVMTypeRef llvm_value_type(struct llvm_ctx *ctx, IDL_tree type)
{
if(type == NULL) return LLVMVoidTypeInContext(ctx->ctx);
switch(IDL_NODE_TYPE(type)) {
case IDLN_TYPE_INTEGER: {
static short bitlens[] = {
[IDL_INTEGER_TYPE_SHORT] = 16,
[IDL_INTEGER_TYPE_LONG] = 32,
[IDL_INTEGER_TYPE_LONGLONG] = 64,
};
int t = IDL_TYPE_INTEGER(type).f_type;
assert(t < G_N_ELEMENTS(bitlens));
return LLVMIntTypeInContext(ctx->ctx, bitlens[t]);
}
case IDLN_NATIVE: {
/* each of these is the size of a single word, which is all LLVM
* wants to know.
*/
if(IS_WORD_TYPE(type) || IS_FPAGE_TYPE(type)
|| IS_TIME_TYPE(type))
{
return ctx->wordt;
} else {
fprintf(stderr, "%s: native type `%s' not supported\n",
__FUNCTION__, NATIVE_NAME(type));
abort();
}
break;
}
case IDLN_TYPE_FLOAT:
switch(IDL_TYPE_FLOAT(type).f_type) {
case IDL_FLOAT_TYPE_FLOAT:
return LLVMFloatTypeInContext(ctx->ctx);
case IDL_FLOAT_TYPE_DOUBLE:
return LLVMDoubleTypeInContext(ctx->ctx);
case IDL_FLOAT_TYPE_LONGDOUBLE:
return LLVMFP128TypeInContext(ctx->ctx);
}
g_assert_not_reached();
case IDLN_TYPE_BOOLEAN:
case IDLN_TYPE_OCTET:
case IDLN_TYPE_CHAR:
return LLVMInt8TypeInContext(ctx->ctx);
case IDLN_TYPE_WIDE_CHAR:
return ctx->i32t;
case IDLN_TYPE_ENUM: return LLVMInt16TypeInContext(ctx->ctx);
default:
NOTDEFINED(type);
}
}
LLVMTypeRef
lp_build_elem_type(struct gallivm_state *gallivm, struct lp_type type)
{
if (type.floating) {
switch(type.width) {
case 32:
return LLVMFloatTypeInContext(gallivm->context);
break;
case 64:
return LLVMDoubleTypeInContext(gallivm->context);
break;
default:
assert(0);
return LLVMFloatTypeInContext(gallivm->context);
}
}
else {
return LLVMIntTypeInContext(gallivm->context, type.width);
}
}
/* Initialize module-independent parts of the context.
*
* The caller is responsible for initializing ctx::module and ctx::builder.
*/
void
ac_llvm_context_init(struct ac_llvm_context *ctx, LLVMContextRef context)
{
LLVMValueRef args[1];
ctx->context = context;
ctx->module = NULL;
ctx->builder = NULL;
ctx->voidt = LLVMVoidTypeInContext(ctx->context);
ctx->i1 = LLVMInt1TypeInContext(ctx->context);
ctx->i8 = LLVMInt8TypeInContext(ctx->context);
ctx->i16 = LLVMIntTypeInContext(ctx->context, 16);
ctx->i32 = LLVMIntTypeInContext(ctx->context, 32);
ctx->i64 = LLVMIntTypeInContext(ctx->context, 64);
ctx->f16 = LLVMHalfTypeInContext(ctx->context);
ctx->f32 = LLVMFloatTypeInContext(ctx->context);
ctx->f64 = LLVMDoubleTypeInContext(ctx->context);
ctx->v4i32 = LLVMVectorType(ctx->i32, 4);
ctx->v4f32 = LLVMVectorType(ctx->f32, 4);
ctx->v8i32 = LLVMVectorType(ctx->i32, 8);
ctx->i32_0 = LLVMConstInt(ctx->i32, 0, false);
ctx->i32_1 = LLVMConstInt(ctx->i32, 1, false);
ctx->f32_0 = LLVMConstReal(ctx->f32, 0.0);
ctx->f32_1 = LLVMConstReal(ctx->f32, 1.0);
ctx->range_md_kind = LLVMGetMDKindIDInContext(ctx->context,
"range", 5);
ctx->invariant_load_md_kind = LLVMGetMDKindIDInContext(ctx->context,
"invariant.load", 14);
ctx->fpmath_md_kind = LLVMGetMDKindIDInContext(ctx->context, "fpmath", 6);
args[0] = LLVMConstReal(ctx->f32, 2.5);
ctx->fpmath_md_2p5_ulp = LLVMMDNodeInContext(ctx->context, args, 1);
ctx->uniform_md_kind = LLVMGetMDKindIDInContext(ctx->context,
"amdgpu.uniform", 14);
ctx->empty_md = LLVMMDNodeInContext(ctx->context, NULL, 0);
}
static LLVMValueRef
emit_fetch_immediate(
struct lp_build_tgsi_context *bld_base,
const struct tgsi_full_src_register *reg,
enum tgsi_opcode_type type,
unsigned swizzle)
{
LLVMTypeRef ctype;
LLVMContextRef ctx = bld_base->base.gallivm->context;
switch (type) {
case TGSI_TYPE_UNSIGNED:
case TGSI_TYPE_SIGNED:
ctype = LLVMInt32TypeInContext(ctx);
break;
case TGSI_TYPE_UNTYPED:
case TGSI_TYPE_FLOAT:
ctype = LLVMFloatTypeInContext(ctx);
break;
default:
ctype = 0;
break;
}
struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
if (swizzle == ~0) {
LLVMValueRef values[TGSI_NUM_CHANNELS] = {};
unsigned chan;
for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
values[chan] = LLVMConstBitCast(bld->immediates[reg->Register.Index][chan], ctype);
}
return lp_build_gather_values(bld_base->base.gallivm, values,
TGSI_NUM_CHANNELS);
} else {
return LLVMConstBitCast(bld->immediates[reg->Register.Index][swizzle], ctype);
}
}
/**
* Pack a single pixel.
*
* @param rgba 4 float vector with the unpacked components.
*
* XXX: This is mostly for reference and testing -- operating a single pixel at
* a time is rarely if ever needed.
*/
LLVMValueRef
lp_build_pack_rgba_aos(struct gallivm_state *gallivm,
const struct util_format_description *desc,
LLVMValueRef rgba)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMTypeRef type;
LLVMValueRef packed = NULL;
LLVMValueRef swizzles[4];
LLVMValueRef shifted, casted, scaled, unswizzled;
LLVMValueRef shifts[4];
LLVMValueRef scales[4];
boolean normalized;
unsigned shift;
unsigned i, j;
assert(desc->layout == UTIL_FORMAT_LAYOUT_PLAIN);
assert(desc->block.width == 1);
assert(desc->block.height == 1);
type = LLVMIntTypeInContext(gallivm->context, desc->block.bits);
/* Unswizzle the color components into the source vector. */
for (i = 0; i < 4; ++i) {
for (j = 0; j < 4; ++j) {
if (desc->swizzle[j] == i)
break;
}
if (j < 4)
swizzles[i] = lp_build_const_int32(gallivm, j);
else
swizzles[i] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
}
unswizzled = LLVMBuildShuffleVector(builder, rgba,
LLVMGetUndef(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4)),
LLVMConstVector(swizzles, 4), "");
normalized = FALSE;
shift = 0;
for (i = 0; i < 4; ++i) {
unsigned bits = desc->channel[i].size;
if (desc->channel[i].type == UTIL_FORMAT_TYPE_VOID) {
shifts[i] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
scales[i] = LLVMGetUndef(LLVMFloatTypeInContext(gallivm->context));
}
else {
unsigned mask = (1 << bits) - 1;
assert(desc->channel[i].type == UTIL_FORMAT_TYPE_UNSIGNED);
assert(bits < 32);
shifts[i] = lp_build_const_int32(gallivm, shift);
if (desc->channel[i].normalized) {
scales[i] = lp_build_const_float(gallivm, mask);
normalized = TRUE;
}
else
scales[i] = lp_build_const_float(gallivm, 1.0);
}
shift += bits;
}
if (normalized)
scaled = LLVMBuildFMul(builder, unswizzled, LLVMConstVector(scales, 4), "");
else
scaled = unswizzled;
casted = LLVMBuildFPToSI(builder, scaled, LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), 4), "");
shifted = LLVMBuildShl(builder, casted, LLVMConstVector(shifts, 4), "");
/* Bitwise or all components */
for (i = 0; i < 4; ++i) {
if (desc->channel[i].type == UTIL_FORMAT_TYPE_UNSIGNED) {
LLVMValueRef component = LLVMBuildExtractElement(builder, shifted,
lp_build_const_int32(gallivm, i), "");
if (packed)
packed = LLVMBuildOr(builder, packed, component, "");
else
packed = component;
}
}
if (!packed)
packed = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
if (desc->block.bits < 32)
packed = LLVMBuildTrunc(builder, packed, type, "");
return packed;
}
/**
* Fetch a pixel into a 4 float AoS.
*
* \param format_desc describes format of the image we're fetching from
* \param ptr address of the pixel block (or the texel if uncompressed)
* \param i, j the sub-block pixel coordinates. For non-compressed formats
* these will always be (0, 0).
* \return a 4 element vector with the pixel's RGBA values.
*/
LLVMValueRef
lp_build_fetch_rgba_aos(struct gallivm_state *gallivm,
const struct util_format_description *format_desc,
struct lp_type type,
LLVMValueRef base_ptr,
LLVMValueRef offset,
LLVMValueRef i,
LLVMValueRef j)
{
LLVMBuilderRef builder = gallivm->builder;
unsigned num_pixels = type.length / 4;
struct lp_build_context bld;
assert(type.length <= LP_MAX_VECTOR_LENGTH);
assert(type.length % 4 == 0);
lp_build_context_init(&bld, gallivm, type);
/*
* Trivial case
*
* The format matches the type (apart of a swizzle) so no need for
* scaling or converting.
*/
if (format_matches_type(format_desc, type) &&
format_desc->block.bits <= type.width * 4 &&
util_is_power_of_two(format_desc->block.bits)) {
LLVMValueRef packed;
/*
* The format matches the type (apart of a swizzle) so no need for
* scaling or converting.
*/
packed = lp_build_gather(gallivm, type.length/4,
format_desc->block.bits, type.width*4,
base_ptr, offset);
assert(format_desc->block.bits <= type.width * type.length);
packed = LLVMBuildBitCast(gallivm->builder, packed,
lp_build_vec_type(gallivm, type), "");
return lp_build_format_swizzle_aos(format_desc, &bld, packed);
}
/*
* Bit arithmetic
*/
if (format_desc->layout == UTIL_FORMAT_LAYOUT_PLAIN &&
(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB ||
format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS) &&
format_desc->block.width == 1 &&
format_desc->block.height == 1 &&
util_is_power_of_two(format_desc->block.bits) &&
format_desc->block.bits <= 32 &&
format_desc->is_bitmask &&
!format_desc->is_mixed &&
(format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED ||
format_desc->channel[1].type == UTIL_FORMAT_TYPE_UNSIGNED)) {
LLVMValueRef tmps[LP_MAX_VECTOR_LENGTH/4];
LLVMValueRef res;
unsigned k;
/*
* Unpack a pixel at a time into a <4 x float> RGBA vector
*/
for (k = 0; k < num_pixels; ++k) {
LLVMValueRef packed;
packed = lp_build_gather_elem(gallivm, num_pixels,
format_desc->block.bits, 32,
base_ptr, offset, k);
tmps[k] = lp_build_unpack_arith_rgba_aos(gallivm,
format_desc,
packed);
}
/*
* Type conversion.
*
* TODO: We could avoid floating conversion for integer to
* integer conversions.
*/
if (gallivm_debug & GALLIVM_DEBUG_PERF && !type.floating) {
debug_printf("%s: unpacking %s with floating point\n",
__FUNCTION__, format_desc->short_name);
}
lp_build_conv(gallivm,
lp_float32_vec4_type(),
type,
tmps, num_pixels, &res, 1);
return lp_build_format_swizzle_aos(format_desc, &bld, res);
}
/*
* YUV / subsampled formats
*/
if (format_desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED) {
struct lp_type tmp_type;
LLVMValueRef tmp;
memset(&tmp_type, 0, sizeof tmp_type);
tmp_type.width = 8;
tmp_type.length = num_pixels * 4;
tmp_type.norm = TRUE;
tmp = lp_build_fetch_subsampled_rgba_aos(gallivm,
format_desc,
num_pixels,
base_ptr,
offset,
i, j);
lp_build_conv(gallivm,
tmp_type, type,
&tmp, 1, &tmp, 1);
return tmp;
}
/*
* Fallback to util_format_description::fetch_rgba_8unorm().
*/
if (format_desc->fetch_rgba_8unorm &&
!type.floating && type.width == 8 && !type.sign && type.norm) {
/*
* Fallback to calling util_format_description::fetch_rgba_8unorm.
*
* This is definitely not the most efficient way of fetching pixels, as
* we miss the opportunity to do vectorization, but this it is a
* convenient for formats or scenarios for which there was no opportunity
* or incentive to optimize.
*/
LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(gallivm->builder)));
char name[256];
LLVMTypeRef i8t = LLVMInt8TypeInContext(gallivm->context);
LLVMTypeRef pi8t = LLVMPointerType(i8t, 0);
LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
LLVMValueRef function;
LLVMValueRef tmp_ptr;
LLVMValueRef tmp;
LLVMValueRef res;
LLVMValueRef callee;
unsigned k;
util_snprintf(name, sizeof name, "util_format_%s_fetch_rgba_8unorm",
format_desc->short_name);
if (gallivm_debug & GALLIVM_DEBUG_PERF) {
debug_printf("%s: falling back to %s\n", __FUNCTION__, name);
}
/*
* Declare and bind format_desc->fetch_rgba_8unorm().
*/
function = LLVMGetNamedFunction(module, name);
if (!function) {
/*
* Function to call looks like:
* fetch(uint8_t *dst, const uint8_t *src, unsigned i, unsigned j)
*/
LLVMTypeRef ret_type;
LLVMTypeRef arg_types[4];
LLVMTypeRef function_type;
ret_type = LLVMVoidTypeInContext(gallivm->context);
arg_types[0] = pi8t;
arg_types[1] = pi8t;
arg_types[2] = i32t;
arg_types[3] = i32t;
function_type = LLVMFunctionType(ret_type, arg_types,
Elements(arg_types), 0);
function = LLVMAddFunction(module, name, function_type);
LLVMSetFunctionCallConv(function, LLVMCCallConv);
LLVMSetLinkage(function, LLVMExternalLinkage);
assert(LLVMIsDeclaration(function));
}
/* make const pointer for the C fetch_rgba_float function */
callee = lp_build_const_int_pointer(gallivm,
func_to_pointer((func_pointer) format_desc->fetch_rgba_8unorm));
/* cast the callee pointer to the function's type */
function = LLVMBuildBitCast(builder, callee,
LLVMTypeOf(function), "cast callee");
tmp_ptr = lp_build_alloca(gallivm, i32t, "");
res = LLVMGetUndef(LLVMVectorType(i32t, num_pixels));
/*
* Invoke format_desc->fetch_rgba_8unorm() for each pixel and insert the result
* in the SoA vectors.
*/
for (k = 0; k < num_pixels; ++k) {
LLVMValueRef index = lp_build_const_int32(gallivm, k);
LLVMValueRef args[4];
args[0] = LLVMBuildBitCast(builder, tmp_ptr, pi8t, "");
args[1] = lp_build_gather_elem_ptr(gallivm, num_pixels,
base_ptr, offset, k);
if (num_pixels == 1) {
args[2] = i;
args[3] = j;
}
else {
args[2] = LLVMBuildExtractElement(builder, i, index, "");
args[3] = LLVMBuildExtractElement(builder, j, index, "");
}
LLVMBuildCall(builder, function, args, Elements(args), "");
tmp = LLVMBuildLoad(builder, tmp_ptr, "");
if (num_pixels == 1) {
res = tmp;
}
else {
res = LLVMBuildInsertElement(builder, res, tmp, index, "");
}
}
/* Bitcast from <n x i32> to <4n x i8> */
res = LLVMBuildBitCast(builder, res, bld.vec_type, "");
return res;
}
/*
* Fallback to util_format_description::fetch_rgba_float().
*/
if (format_desc->fetch_rgba_float) {
/*
* Fallback to calling util_format_description::fetch_rgba_float.
*
* This is definitely not the most efficient way of fetching pixels, as
* we miss the opportunity to do vectorization, but this it is a
* convenient for formats or scenarios for which there was no opportunity
* or incentive to optimize.
*/
LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
char name[256];
LLVMTypeRef f32t = LLVMFloatTypeInContext(gallivm->context);
LLVMTypeRef f32x4t = LLVMVectorType(f32t, 4);
LLVMTypeRef pf32t = LLVMPointerType(f32t, 0);
LLVMTypeRef pi8t = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
LLVMValueRef function;
LLVMValueRef tmp_ptr;
LLVMValueRef tmps[LP_MAX_VECTOR_LENGTH/4];
LLVMValueRef res;
LLVMValueRef callee;
unsigned k;
util_snprintf(name, sizeof name, "util_format_%s_fetch_rgba_float",
format_desc->short_name);
if (gallivm_debug & GALLIVM_DEBUG_PERF) {
debug_printf("%s: falling back to %s\n", __FUNCTION__, name);
}
/*
* Declare and bind format_desc->fetch_rgba_float().
*/
function = LLVMGetNamedFunction(module, name);
if (!function) {
/*
* Function to call looks like:
* fetch(float *dst, const uint8_t *src, unsigned i, unsigned j)
*/
LLVMTypeRef ret_type;
LLVMTypeRef arg_types[4];
LLVMTypeRef function_type;
ret_type = LLVMVoidTypeInContext(gallivm->context);
arg_types[0] = pf32t;
arg_types[1] = pi8t;
arg_types[2] = i32t;
arg_types[3] = i32t;
function_type = LLVMFunctionType(ret_type, arg_types,
Elements(arg_types), 0);
function = LLVMAddFunction(module, name, function_type);
LLVMSetFunctionCallConv(function, LLVMCCallConv);
LLVMSetLinkage(function, LLVMExternalLinkage);
assert(LLVMIsDeclaration(function));
}
/* Note: we're using this casting here instead of LLVMAddGlobalMapping()
* to work around a bug in LLVM 2.6.
*/
/* make const pointer for the C fetch_rgba_float function */
callee = lp_build_const_int_pointer(gallivm,
func_to_pointer((func_pointer) format_desc->fetch_rgba_float));
/* cast the callee pointer to the function's type */
function = LLVMBuildBitCast(builder, callee,
LLVMTypeOf(function), "cast callee");
tmp_ptr = lp_build_alloca(gallivm, f32x4t, "");
/*
* Invoke format_desc->fetch_rgba_float() for each pixel and insert the result
* in the SoA vectors.
*/
for (k = 0; k < num_pixels; ++k) {
LLVMValueRef args[4];
args[0] = LLVMBuildBitCast(builder, tmp_ptr, pf32t, "");
args[1] = lp_build_gather_elem_ptr(gallivm, num_pixels,
base_ptr, offset, k);
if (num_pixels == 1) {
args[2] = i;
args[3] = j;
}
else {
LLVMValueRef index = lp_build_const_int32(gallivm, k);
args[2] = LLVMBuildExtractElement(builder, i, index, "");
args[3] = LLVMBuildExtractElement(builder, j, index, "");
}
LLVMBuildCall(builder, function, args, Elements(args), "");
tmps[k] = LLVMBuildLoad(builder, tmp_ptr, "");
}
lp_build_conv(gallivm,
lp_float32_vec4_type(),
type,
tmps, num_pixels, &res, 1);
return res;
}
assert(0);
return lp_build_undef(gallivm, type);
}
/**
* Increment the shader input attribute values.
* This is called when we move from one quad to the next.
*/
static void
attribs_update(struct lp_build_interp_soa_context *bld,
struct gallivm_state *gallivm,
LLVMValueRef loop_iter,
int start,
int end)
{
LLVMBuilderRef builder = gallivm->builder;
struct lp_build_context *coeff_bld = &bld->coeff_bld;
LLVMValueRef oow = NULL;
unsigned attrib;
unsigned chan;
for(attrib = start; attrib < end; ++attrib) {
const unsigned mask = bld->mask[attrib];
const unsigned interp = bld->interp[attrib];
for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
if(mask & (1 << chan)) {
LLVMValueRef a;
if (interp == LP_INTERP_CONSTANT ||
interp == LP_INTERP_FACING) {
a = LLVMBuildLoad(builder, bld->a[attrib][chan], "");
}
else if (interp == LP_INTERP_POSITION) {
assert(attrib > 0);
a = bld->attribs[0][chan];
}
else {
LLVMValueRef dadq;
a = bld->a[attrib][chan];
/*
* Broadcast the attribute value for this quad into all elements
*/
{
/* stored as vector load as float */
LLVMTypeRef ptr_type = LLVMPointerType(LLVMFloatTypeInContext(
gallivm->context), 0);
LLVMValueRef ptr;
a = LLVMBuildBitCast(builder, a, ptr_type, "");
ptr = LLVMBuildGEP(builder, a, &loop_iter, 1, "");
a = LLVMBuildLoad(builder, ptr, "");
a = lp_build_broadcast_scalar(&bld->coeff_bld, a);
}
/*
* Get the derivatives.
*/
dadq = bld->dadq[attrib][chan];
#if PERSPECTIVE_DIVIDE_PER_QUAD
if (interp == LP_INTERP_PERSPECTIVE) {
LLVMValueRef dwdq = bld->dadq[0][3];
if (oow == NULL) {
assert(bld->oow);
oow = LLVMBuildShuffleVector(coeff_bld->builder,
bld->oow, coeff_bld->undef,
shuffle, "");
}
dadq = lp_build_sub(coeff_bld,
dadq,
lp_build_mul(coeff_bld, a, dwdq));
dadq = lp_build_mul(coeff_bld, dadq, oow);
}
#endif
/*
* Add the derivatives
*/
a = lp_build_add(coeff_bld, a, dadq);
#if !PERSPECTIVE_DIVIDE_PER_QUAD
if (interp == LP_INTERP_PERSPECTIVE) {
if (oow == NULL) {
LLVMValueRef w = bld->attribs[0][3];
assert(attrib != 0);
assert(bld->mask[0] & TGSI_WRITEMASK_W);
oow = lp_build_rcp(coeff_bld, w);
}
a = lp_build_mul(coeff_bld, a, oow);
}
#endif
if (attrib == 0 && chan == 2) {
/* FIXME: Depth values can exceed 1.0, due to the fact that
* setup interpolation coefficients refer to (0,0) which causes
* precision loss. So we must clamp to 1.0 here to avoid artifacts
*/
a = lp_build_min(coeff_bld, a, coeff_bld->one);
}
attrib_name(a, attrib, chan, "");
}
bld->attribs[attrib][chan] = a;
}
}
}
}
/**
* Unpack a single pixel into its RGBA components.
*
* @param desc the pixel format for the packed pixel value
* @param packed integer pixel in a format such as PIPE_FORMAT_B8G8R8A8_UNORM
*
* @return RGBA in a float[4] or ubyte[4] or ushort[4] vector.
*/
static INLINE LLVMValueRef
lp_build_unpack_arith_rgba_aos(struct gallivm_state *gallivm,
const struct util_format_description *desc,
LLVMValueRef packed)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef shifted, casted, scaled, masked;
LLVMValueRef shifts[4];
LLVMValueRef masks[4];
LLVMValueRef scales[4];
boolean normalized;
boolean needs_uitofp;
unsigned shift;
unsigned i;
/* TODO: Support more formats */
assert(desc->layout == UTIL_FORMAT_LAYOUT_PLAIN);
assert(desc->block.width == 1);
assert(desc->block.height == 1);
assert(desc->block.bits <= 32);
/* Do the intermediate integer computations with 32bit integers since it
* matches floating point size */
assert (LLVMTypeOf(packed) == LLVMInt32TypeInContext(gallivm->context));
/* Broadcast the packed value to all four channels
* before: packed = BGRA
* after: packed = {BGRA, BGRA, BGRA, BGRA}
*/
packed = LLVMBuildInsertElement(builder,
LLVMGetUndef(LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), 4)),
packed,
LLVMConstNull(LLVMInt32TypeInContext(gallivm->context)),
"");
packed = LLVMBuildShuffleVector(builder,
packed,
LLVMGetUndef(LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), 4)),
LLVMConstNull(LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), 4)),
"");
/* Initialize vector constants */
normalized = FALSE;
needs_uitofp = FALSE;
shift = 0;
/* Loop over 4 color components */
for (i = 0; i < 4; ++i) {
unsigned bits = desc->channel[i].size;
if (desc->channel[i].type == UTIL_FORMAT_TYPE_VOID) {
shifts[i] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
masks[i] = LLVMConstNull(LLVMInt32TypeInContext(gallivm->context));
scales[i] = LLVMConstNull(LLVMFloatTypeInContext(gallivm->context));
}
else {
unsigned long long mask = (1ULL << bits) - 1;
assert(desc->channel[i].type == UTIL_FORMAT_TYPE_UNSIGNED);
if (bits == 32) {
needs_uitofp = TRUE;
}
shifts[i] = lp_build_const_int32(gallivm, shift);
masks[i] = lp_build_const_int32(gallivm, mask);
if (desc->channel[i].normalized) {
scales[i] = lp_build_const_float(gallivm, 1.0 / mask);
normalized = TRUE;
}
else
scales[i] = lp_build_const_float(gallivm, 1.0);
}
shift += bits;
}
/* Ex: convert packed = {BGRA, BGRA, BGRA, BGRA}
* into masked = {B, G, R, A}
*/
shifted = LLVMBuildLShr(builder, packed, LLVMConstVector(shifts, 4), "");
masked = LLVMBuildAnd(builder, shifted, LLVMConstVector(masks, 4), "");
if (!needs_uitofp) {
/* UIToFP can't be expressed in SSE2 */
casted = LLVMBuildSIToFP(builder, masked, LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), "");
} else {
casted = LLVMBuildUIToFP(builder, masked, LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), "");
}
/* At this point 'casted' may be a vector of floats such as
* {255.0, 255.0, 255.0, 255.0}. Next, if the pixel values are normalized
* we'll scale this to {1.0, 1.0, 1.0, 1.0}.
*/
if (normalized)
scaled = LLVMBuildFMul(builder, casted, LLVMConstVector(scales, 4), "");
else
scaled = casted;
return scaled;
}
static void
lp_jit_create_types(struct lp_fragment_shader_variant *lp)
{
struct gallivm_state *gallivm = lp->gallivm;
LLVMContextRef lc = gallivm->context;
LLVMTypeRef viewport_type, texture_type, sampler_type;
/* struct lp_jit_viewport */
{
LLVMTypeRef elem_types[LP_JIT_VIEWPORT_NUM_FIELDS];
elem_types[LP_JIT_VIEWPORT_MIN_DEPTH] =
elem_types[LP_JIT_VIEWPORT_MAX_DEPTH] = LLVMFloatTypeInContext(lc);
viewport_type = LLVMStructTypeInContext(lc, elem_types,
Elements(elem_types), 0);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_viewport, min_depth,
gallivm->target, viewport_type,
LP_JIT_VIEWPORT_MIN_DEPTH);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_viewport, max_depth,
gallivm->target, viewport_type,
LP_JIT_VIEWPORT_MAX_DEPTH);
LP_CHECK_STRUCT_SIZE(struct lp_jit_viewport,
gallivm->target, viewport_type);
}
/* struct lp_jit_texture */
{
LLVMTypeRef elem_types[LP_JIT_TEXTURE_NUM_FIELDS];
elem_types[LP_JIT_TEXTURE_WIDTH] =
elem_types[LP_JIT_TEXTURE_HEIGHT] =
elem_types[LP_JIT_TEXTURE_DEPTH] =
elem_types[LP_JIT_TEXTURE_FIRST_LEVEL] =
elem_types[LP_JIT_TEXTURE_LAST_LEVEL] = LLVMInt32TypeInContext(lc);
elem_types[LP_JIT_TEXTURE_BASE] = LLVMPointerType(LLVMInt8TypeInContext(lc), 0);
elem_types[LP_JIT_TEXTURE_ROW_STRIDE] =
elem_types[LP_JIT_TEXTURE_IMG_STRIDE] =
elem_types[LP_JIT_TEXTURE_MIP_OFFSETS] =
LLVMArrayType(LLVMInt32TypeInContext(lc), LP_MAX_TEXTURE_LEVELS);
texture_type = LLVMStructTypeInContext(lc, elem_types,
Elements(elem_types), 0);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, width,
gallivm->target, texture_type,
LP_JIT_TEXTURE_WIDTH);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, height,
gallivm->target, texture_type,
LP_JIT_TEXTURE_HEIGHT);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, depth,
gallivm->target, texture_type,
LP_JIT_TEXTURE_DEPTH);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, first_level,
gallivm->target, texture_type,
LP_JIT_TEXTURE_FIRST_LEVEL);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, last_level,
gallivm->target, texture_type,
LP_JIT_TEXTURE_LAST_LEVEL);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, base,
gallivm->target, texture_type,
LP_JIT_TEXTURE_BASE);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, row_stride,
gallivm->target, texture_type,
LP_JIT_TEXTURE_ROW_STRIDE);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, img_stride,
gallivm->target, texture_type,
LP_JIT_TEXTURE_IMG_STRIDE);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, mip_offsets,
gallivm->target, texture_type,
LP_JIT_TEXTURE_MIP_OFFSETS);
LP_CHECK_STRUCT_SIZE(struct lp_jit_texture,
gallivm->target, texture_type);
}
/* struct lp_jit_sampler */
{
LLVMTypeRef elem_types[LP_JIT_SAMPLER_NUM_FIELDS];
elem_types[LP_JIT_SAMPLER_MIN_LOD] =
elem_types[LP_JIT_SAMPLER_MAX_LOD] =
elem_types[LP_JIT_SAMPLER_LOD_BIAS] = LLVMFloatTypeInContext(lc);
elem_types[LP_JIT_SAMPLER_BORDER_COLOR] =
LLVMArrayType(LLVMFloatTypeInContext(lc), 4);
sampler_type = LLVMStructTypeInContext(lc, elem_types,
Elements(elem_types), 0);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, min_lod,
gallivm->target, sampler_type,
LP_JIT_SAMPLER_MIN_LOD);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, max_lod,
gallivm->target, sampler_type,
LP_JIT_SAMPLER_MAX_LOD);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, lod_bias,
gallivm->target, sampler_type,
LP_JIT_SAMPLER_LOD_BIAS);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, border_color,
gallivm->target, sampler_type,
LP_JIT_SAMPLER_BORDER_COLOR);
LP_CHECK_STRUCT_SIZE(struct lp_jit_sampler,
gallivm->target, sampler_type);
}
/* struct lp_jit_context */
{
LLVMTypeRef elem_types[LP_JIT_CTX_COUNT];
LLVMTypeRef context_type;
elem_types[LP_JIT_CTX_CONSTANTS] =
LLVMArrayType(LLVMPointerType(LLVMFloatTypeInContext(lc), 0), LP_MAX_TGSI_CONST_BUFFERS);
elem_types[LP_JIT_CTX_NUM_CONSTANTS] =
LLVMArrayType(LLVMInt32TypeInContext(lc), LP_MAX_TGSI_CONST_BUFFERS);
elem_types[LP_JIT_CTX_ALPHA_REF] = LLVMFloatTypeInContext(lc);
elem_types[LP_JIT_CTX_STENCIL_REF_FRONT] =
elem_types[LP_JIT_CTX_STENCIL_REF_BACK] = LLVMInt32TypeInContext(lc);
elem_types[LP_JIT_CTX_U8_BLEND_COLOR] = LLVMPointerType(LLVMInt8TypeInContext(lc), 0);
elem_types[LP_JIT_CTX_F_BLEND_COLOR] = LLVMPointerType(LLVMFloatTypeInContext(lc), 0);
elem_types[LP_JIT_CTX_VIEWPORTS] = LLVMPointerType(viewport_type, 0);
elem_types[LP_JIT_CTX_TEXTURES] = LLVMArrayType(texture_type,
PIPE_MAX_SHADER_SAMPLER_VIEWS);
elem_types[LP_JIT_CTX_SAMPLERS] = LLVMArrayType(sampler_type,
PIPE_MAX_SAMPLERS);
context_type = LLVMStructTypeInContext(lc, elem_types,
Elements(elem_types), 0);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, constants,
gallivm->target, context_type,
LP_JIT_CTX_CONSTANTS);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, num_constants,
gallivm->target, context_type,
LP_JIT_CTX_NUM_CONSTANTS);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, alpha_ref_value,
gallivm->target, context_type,
LP_JIT_CTX_ALPHA_REF);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, stencil_ref_front,
gallivm->target, context_type,
LP_JIT_CTX_STENCIL_REF_FRONT);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, stencil_ref_back,
gallivm->target, context_type,
LP_JIT_CTX_STENCIL_REF_BACK);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, u8_blend_color,
gallivm->target, context_type,
LP_JIT_CTX_U8_BLEND_COLOR);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, f_blend_color,
gallivm->target, context_type,
LP_JIT_CTX_F_BLEND_COLOR);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, viewports,
gallivm->target, context_type,
LP_JIT_CTX_VIEWPORTS);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, textures,
gallivm->target, context_type,
LP_JIT_CTX_TEXTURES);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, samplers,
gallivm->target, context_type,
LP_JIT_CTX_SAMPLERS);
LP_CHECK_STRUCT_SIZE(struct lp_jit_context,
gallivm->target, context_type);
lp->jit_context_ptr_type = LLVMPointerType(context_type, 0);
}
/* struct lp_jit_thread_data */
{
LLVMTypeRef elem_types[LP_JIT_THREAD_DATA_COUNT];
LLVMTypeRef thread_data_type;
elem_types[LP_JIT_THREAD_DATA_COUNTER] = LLVMInt64TypeInContext(lc);
elem_types[LP_JIT_THREAD_DATA_RASTER_STATE_VIEWPORT_INDEX] =
LLVMInt32TypeInContext(lc);
thread_data_type = LLVMStructTypeInContext(lc, elem_types,
Elements(elem_types), 0);
lp->jit_thread_data_ptr_type = LLVMPointerType(thread_data_type, 0);
}
if (gallivm_debug & GALLIVM_DEBUG_IR) {
LLVMDumpModule(gallivm->module);
}
}
LLVMTypeRef ett_llvm_type(EagleComplexType *type)
{
switch(type->type)
{
case ETVoid:
return LLVMVoidTypeInContext(utl_get_current_context());
case ETFloat:
return LLVMFloatTypeInContext(utl_get_current_context());
case ETDouble:
return LLVMDoubleTypeInContext(utl_get_current_context());
case ETInt1:
return LLVMInt1TypeInContext(utl_get_current_context());
case ETGeneric: // In practice this doesn't matter
case ETAny:
case ETInt8:
case ETUInt8:
return LLVMInt8TypeInContext(utl_get_current_context());
case ETInt16:
case ETUInt16:
return LLVMInt16TypeInContext(utl_get_current_context());
case ETInt32:
case ETUInt32:
return LLVMInt32TypeInContext(utl_get_current_context());
case ETInt64:
case ETUInt64:
return LLVMInt64TypeInContext(utl_get_current_context());
case ETCString:
return LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
case ETEnum:
return LLVMInt64TypeInContext(utl_get_current_context());
case ETGenerator:
{
if(generator_type)
return generator_type;
LLVMTypeRef ptmp[2];
ptmp[0] = LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
ptmp[1] = LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
generator_type = LLVMStructCreateNamed(utl_get_current_context(), "__egl_gen_strct");
LLVMStructSetBody(generator_type, ptmp, 2, 0);
return generator_type;
}
case ETClass:
case ETStruct:
{
EagleStructType *st = (EagleStructType *)type;
LLVMTypeRef loaded = LLVMGetTypeByName(the_module, st->name);
if(loaded)
return loaded;
return NULL;
// LLVMTypeRef ty = LLVMStructTypeInContext(utl_get_current_context(),
}
case ETInterface:
{
return LLVMInt8TypeInContext(utl_get_current_context());
}
case ETPointer:
{
EaglePointerType *pt = (EaglePointerType *)type;
if(pt->counted || pt->weak)
{
LLVMTypeRef ptmp[2];
ptmp[0] = LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
ptmp[1] = LLVMInt1TypeInContext(utl_get_current_context());
LLVMTypeRef tys[6];
tys[0] = LLVMInt64TypeInContext(utl_get_current_context());
tys[1] = LLVMInt16TypeInContext(utl_get_current_context());
tys[2] = LLVMInt16TypeInContext(utl_get_current_context());
tys[3] = LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
tys[4] = LLVMPointerType(LLVMFunctionType(LLVMVoidTypeInContext(utl_get_current_context()), ptmp, 2, 0), 0);
tys[5] = ett_llvm_type(pt->to);
return LLVMPointerType(ty_get_counted(LLVMStructTypeInContext(utl_get_current_context(), tys, 6, 0)), 0);
}
return LLVMPointerType(ett_llvm_type(((EaglePointerType *)type)->to), 0);
}
case ETArray:
{
EagleArrayType *at = (EagleArrayType *)type;
if(at->ct < 0)
return LLVMPointerType(ett_llvm_type(at->of), 0);
else
return LLVMArrayType(ett_llvm_type(at->of), at->ct);
}
case ETFunction:
{
EagleFunctionType *ft = (EagleFunctionType *)type;
if(ET_IS_CLOSURE(type))
{
LLVMTypeRef tys[2];
tys[0] = LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
tys[1] = LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
return LLVMStructTypeInContext(utl_get_current_context(), tys, 2, 0);
}
LLVMTypeRef *tys = malloc(sizeof(LLVMTypeRef) * ft->pct);
int i;
for(i = 0; i < ft->pct; i++)
tys[i] = ett_llvm_type(ft->params[i]);
LLVMTypeRef out = LLVMFunctionType(ett_llvm_type(ft->retType), tys, ft->pct, 0);
free(tys);
return out;
}
default:
return NULL;
}
}
static void init_runtime(compile_t* c)
{
c->str_1 = stringtab("$1");
c->str_Bool = stringtab("Bool");
c->str_I8 = stringtab("I8");
c->str_I16 = stringtab("I16");
c->str_I32 = stringtab("I32");
c->str_I64 = stringtab("I64");
c->str_I128 = stringtab("I128");
c->str_U8 = stringtab("U8");
c->str_U16 = stringtab("U16");
c->str_U32 = stringtab("U32");
c->str_U64 = stringtab("U64");
c->str_U128 = stringtab("U128");
c->str_F32 = stringtab("F32");
c->str_F64 = stringtab("F64");
c->str_Pointer = stringtab("Pointer");
c->str_Array = stringtab("Array");
c->str_Platform = stringtab("Platform");
c->str_add = stringtab("add");
c->str_sub = stringtab("sub");
c->str_mul = stringtab("mul");
c->str_div = stringtab("div");
c->str_mod = stringtab("mod");
c->str_neg = stringtab("neg");
c->str_and = stringtab("op_and");
c->str_or = stringtab("op_or");
c->str_xor = stringtab("op_xor");
c->str_not = stringtab("op_not");
c->str_shl = stringtab("shl");
c->str_shr = stringtab("shr");
c->str_eq = stringtab("eq");
c->str_ne = stringtab("ne");
c->str_lt = stringtab("lt");
c->str_le = stringtab("le");
c->str_ge = stringtab("ge");
c->str_gt = stringtab("gt");
LLVMTypeRef type;
LLVMTypeRef params[4];
LLVMValueRef value;
c->void_type = LLVMVoidTypeInContext(c->context);
c->i1 = LLVMInt1TypeInContext(c->context);
c->i8 = LLVMInt8TypeInContext(c->context);
c->i16 = LLVMInt16TypeInContext(c->context);
c->i32 = LLVMInt32TypeInContext(c->context);
c->i64 = LLVMInt64TypeInContext(c->context);
c->i128 = LLVMIntTypeInContext(c->context, 128);
c->f32 = LLVMFloatTypeInContext(c->context);
c->f64 = LLVMDoubleTypeInContext(c->context);
c->intptr = LLVMIntPtrTypeInContext(c->context, c->target_data);
// i8*
c->void_ptr = LLVMPointerType(c->i8, 0);
// forward declare object
c->object_type = LLVMStructCreateNamed(c->context, "$object");
c->object_ptr = LLVMPointerType(c->object_type, 0);
// padding required in an actor between the descriptor and fields
c->actor_pad = LLVMArrayType(c->i8, PONY_ACTOR_PAD_SIZE);
// message
params[0] = c->i32; // size
params[1] = c->i32; // id
c->msg_type = LLVMStructCreateNamed(c->context, "$message");
c->msg_ptr = LLVMPointerType(c->msg_type, 0);
LLVMStructSetBody(c->msg_type, params, 2, false);
// trace
// void (*)($object*)
params[0] = c->object_ptr;
c->trace_type = LLVMFunctionType(c->void_type, params, 1, false);
c->trace_fn = LLVMPointerType(c->trace_type, 0);
// dispatch
// void (*)($object*, $message*)
params[0] = c->object_ptr;
params[1] = c->msg_ptr;
c->dispatch_type = LLVMFunctionType(c->void_type, params, 2, false);
c->dispatch_fn = LLVMPointerType(c->dispatch_type, 0);
// void (*)($object*)
params[0] = c->object_ptr;
c->final_fn = LLVMPointerType(
LLVMFunctionType(c->void_type, params, 1, false), 0);
// descriptor, opaque version
// We need this in order to build our own structure.
const char* desc_name = genname_descriptor(NULL);
c->descriptor_type = LLVMStructCreateNamed(c->context, desc_name);
c->descriptor_ptr = LLVMPointerType(c->descriptor_type, 0);
// field descriptor
// Also needed to build a descriptor structure.
params[0] = c->i32;
params[1] = c->descriptor_ptr;
c->field_descriptor = LLVMStructTypeInContext(c->context, params, 2, false);
// descriptor, filled in
c->descriptor_type = gendesc_type(c, NULL);
// define object
params[0] = c->descriptor_ptr;
LLVMStructSetBody(c->object_type, params, 1, false);
// $object* pony_create($desc*)
params[0] = c->descriptor_ptr;
type = LLVMFunctionType(c->object_ptr, params, 1, false);
value = LLVMAddFunction(c->module, "pony_create", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMSetReturnNoAlias(value);
// void pony_destroy($object*)
params[0] = c->object_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_destroy", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
//LLVMSetReturnNoAlias(value);
// void pony_sendv($object*, $message*);
params[0] = c->object_ptr;
params[1] = c->msg_ptr;
type = LLVMFunctionType(c->void_type, params, 2, false);
value = LLVMAddFunction(c->module, "pony_sendv", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// i8* pony_alloc(i64)
params[0] = c->i64;
type = LLVMFunctionType(c->void_ptr, params, 1, false);
value = LLVMAddFunction(c->module, "pony_alloc", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMSetReturnNoAlias(value);
// i8* pony_realloc(i8*, i64)
params[0] = c->void_ptr;
params[1] = c->i64;
type = LLVMFunctionType(c->void_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_realloc", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMSetReturnNoAlias(value);
// i8* pony_alloc_final(i64, c->final_fn)
params[0] = c->i64;
params[1] = c->final_fn;
type = LLVMFunctionType(c->void_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc_final", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMSetReturnNoAlias(value);
// $message* pony_alloc_msg(i32, i32)
params[0] = c->i32;
params[1] = c->i32;
type = LLVMFunctionType(c->msg_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc_msg", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMSetReturnNoAlias(value);
// void pony_trace(i8*)
params[0] = c->void_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_trace", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_traceactor($object*)
params[0] = c->object_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_traceactor", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_traceobject($object*, trace_fn)
params[0] = c->object_ptr;
params[1] = c->trace_fn;
type = LLVMFunctionType(c->void_type, params, 2, false);
value = LLVMAddFunction(c->module, "pony_traceobject", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_traceunknown($object*)
params[0] = c->object_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_traceunknown", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_trace_tag_or_actor($object*)
params[0] = c->object_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_trace_tag_or_actor", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_gc_send()
type = LLVMFunctionType(c->void_type, NULL, 0, false);
value = LLVMAddFunction(c->module, "pony_gc_send", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_gc_recv()
type = LLVMFunctionType(c->void_type, NULL, 0, false);
value = LLVMAddFunction(c->module, "pony_gc_recv", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_send_done()
type = LLVMFunctionType(c->void_type, NULL, 0, false);
value = LLVMAddFunction(c->module, "pony_send_done", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_recv_done()
type = LLVMFunctionType(c->void_type, NULL, 0, false);
value = LLVMAddFunction(c->module, "pony_recv_done", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// i32 pony_init(i32, i8**)
params[0] = c->i32;
params[1] = LLVMPointerType(c->void_ptr, 0);
type = LLVMFunctionType(c->i32, params, 2, false);
value = LLVMAddFunction(c->module, "pony_init", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_become($object*)
params[0] = c->object_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_become", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// i32 pony_start(i32)
params[0] = c->i32;
type = LLVMFunctionType(c->i32, params, 1, false);
value = LLVMAddFunction(c->module, "pony_start", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_throw()
type = LLVMFunctionType(c->void_type, NULL, 0, false);
LLVMAddFunction(c->module, "pony_throw", type);
// i32 pony_personality_v0(...)
type = LLVMFunctionType(c->i32, NULL, 0, true);
c->personality = LLVMAddFunction(c->module, "pony_personality_v0", type);
// i8* memcpy(...)
type = LLVMFunctionType(c->void_ptr, NULL, 0, true);
value = LLVMAddFunction(c->module, "memcpy", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// i8* memmove(...)
type = LLVMFunctionType(c->void_ptr, NULL, 0, true);
value = LLVMAddFunction(c->module, "memmove", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
}
static void init_runtime(compile_t* c)
{
c->str_builtin = stringtab("$0");
c->str_Bool = stringtab("Bool");
c->str_I8 = stringtab("I8");
c->str_I16 = stringtab("I16");
c->str_I32 = stringtab("I32");
c->str_I64 = stringtab("I64");
c->str_I128 = stringtab("I128");
c->str_ILong = stringtab("ILong");
c->str_ISize = stringtab("ISize");
c->str_U8 = stringtab("U8");
c->str_U16 = stringtab("U16");
c->str_U32 = stringtab("U32");
c->str_U64 = stringtab("U64");
c->str_U128 = stringtab("U128");
c->str_ULong = stringtab("ULong");
c->str_USize = stringtab("USize");
c->str_F32 = stringtab("F32");
c->str_F64 = stringtab("F64");
c->str_Pointer = stringtab("Pointer");
c->str_Maybe = stringtab("MaybePointer");
c->str_DoNotOptimise = stringtab("DoNotOptimise");
c->str_Array = stringtab("Array");
c->str_String = stringtab("String");
c->str_Platform = stringtab("Platform");
c->str_Main = stringtab("Main");
c->str_Env = stringtab("Env");
c->str_add = stringtab("add");
c->str_sub = stringtab("sub");
c->str_mul = stringtab("mul");
c->str_div = stringtab("div");
c->str_mod = stringtab("mod");
c->str_neg = stringtab("neg");
c->str_add_unsafe = stringtab("add_unsafe");
c->str_sub_unsafe = stringtab("sub_unsafe");
c->str_mul_unsafe = stringtab("mul_unsafe");
c->str_div_unsafe = stringtab("div_unsafe");
c->str_mod_unsafe = stringtab("mod_unsafe");
c->str_neg_unsafe = stringtab("neg_unsafe");
c->str_and = stringtab("op_and");
c->str_or = stringtab("op_or");
c->str_xor = stringtab("op_xor");
c->str_not = stringtab("op_not");
c->str_shl = stringtab("shl");
c->str_shr = stringtab("shr");
c->str_shl_unsafe = stringtab("shl_unsafe");
c->str_shr_unsafe = stringtab("shr_unsafe");
c->str_eq = stringtab("eq");
c->str_ne = stringtab("ne");
c->str_lt = stringtab("lt");
c->str_le = stringtab("le");
c->str_ge = stringtab("ge");
c->str_gt = stringtab("gt");
c->str_eq_unsafe = stringtab("eq_unsafe");
c->str_ne_unsafe = stringtab("ne_unsafe");
c->str_lt_unsafe = stringtab("lt_unsafe");
c->str_le_unsafe = stringtab("le_unsafe");
c->str_ge_unsafe = stringtab("ge_unsafe");
c->str_gt_unsafe = stringtab("gt_unsafe");
c->str_this = stringtab("this");
c->str_create = stringtab("create");
c->str__create = stringtab("_create");
c->str__init = stringtab("_init");
c->str__final = stringtab("_final");
c->str__event_notify = stringtab("_event_notify");
c->str__serialise_space = stringtab("_serialise_space");
c->str__serialise = stringtab("_serialise");
c->str__deserialise = stringtab("_deserialise");
LLVMTypeRef type;
LLVMTypeRef params[5];
LLVMValueRef value;
c->void_type = LLVMVoidTypeInContext(c->context);
c->i1 = LLVMInt1TypeInContext(c->context);
c->i8 = LLVMInt8TypeInContext(c->context);
c->i16 = LLVMInt16TypeInContext(c->context);
c->i32 = LLVMInt32TypeInContext(c->context);
c->i64 = LLVMInt64TypeInContext(c->context);
c->i128 = LLVMIntTypeInContext(c->context, 128);
c->f32 = LLVMFloatTypeInContext(c->context);
c->f64 = LLVMDoubleTypeInContext(c->context);
c->intptr = LLVMIntPtrTypeInContext(c->context, c->target_data);
// i8*
c->void_ptr = LLVMPointerType(c->i8, 0);
// forward declare object
c->object_type = LLVMStructCreateNamed(c->context, "__object");
c->object_ptr = LLVMPointerType(c->object_type, 0);
// padding required in an actor between the descriptor and fields
c->actor_pad = LLVMArrayType(c->i8, PONY_ACTOR_PAD_SIZE);
// message
params[0] = c->i32; // size
params[1] = c->i32; // id
c->msg_type = LLVMStructCreateNamed(c->context, "__message");
c->msg_ptr = LLVMPointerType(c->msg_type, 0);
LLVMStructSetBody(c->msg_type, params, 2, false);
// trace
// void (*)(i8*, __object*)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
c->trace_type = LLVMFunctionType(c->void_type, params, 2, false);
c->trace_fn = LLVMPointerType(c->trace_type, 0);
// serialise
// void (*)(i8*, __object*, i8*, intptr, i32)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
params[2] = c->void_ptr;
params[3] = c->intptr;
params[4] = c->i32;
c->serialise_type = LLVMFunctionType(c->void_type, params, 5, false);
c->serialise_fn = LLVMPointerType(c->serialise_type, 0);
// serialise_space
// i64 (__object*)
params[0] = c->object_ptr;
c->custom_serialise_space_fn = LLVMPointerType(
LLVMFunctionType(c->i64, params, 1, false), 0);
// custom_deserialise
// void (*)(__object*, void*)
params[0] = c->object_ptr;
params[1] = c->void_ptr;
c->custom_deserialise_fn = LLVMPointerType(
LLVMFunctionType(c->void_type, params, 2, false), 0);
// dispatch
// void (*)(i8*, __object*, $message*)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
params[2] = c->msg_ptr;
c->dispatch_type = LLVMFunctionType(c->void_type, params, 3, false);
c->dispatch_fn = LLVMPointerType(c->dispatch_type, 0);
// void (*)(__object*)
params[0] = c->object_ptr;
c->final_fn = LLVMPointerType(
LLVMFunctionType(c->void_type, params, 1, false), 0);
// descriptor, opaque version
// We need this in order to build our own structure.
const char* desc_name = genname_descriptor(NULL);
c->descriptor_type = LLVMStructCreateNamed(c->context, desc_name);
c->descriptor_ptr = LLVMPointerType(c->descriptor_type, 0);
// field descriptor
// Also needed to build a descriptor structure.
params[0] = c->i32;
params[1] = c->descriptor_ptr;
c->field_descriptor = LLVMStructTypeInContext(c->context, params, 2, false);
// descriptor, filled in
gendesc_basetype(c, c->descriptor_type);
// define object
params[0] = c->descriptor_ptr;
LLVMStructSetBody(c->object_type, params, 1, false);
#if PONY_LLVM >= 309
LLVM_DECLARE_ATTRIBUTEREF(nounwind_attr, nounwind, 0);
LLVM_DECLARE_ATTRIBUTEREF(readnone_attr, readnone, 0);
LLVM_DECLARE_ATTRIBUTEREF(readonly_attr, readonly, 0);
LLVM_DECLARE_ATTRIBUTEREF(inacc_or_arg_mem_attr,
inaccessiblemem_or_argmemonly, 0);
LLVM_DECLARE_ATTRIBUTEREF(noalias_attr, noalias, 0);
LLVM_DECLARE_ATTRIBUTEREF(noreturn_attr, noreturn, 0);
LLVM_DECLARE_ATTRIBUTEREF(deref_actor_attr, dereferenceable,
PONY_ACTOR_PAD_SIZE + (target_is_ilp32(c->opt->triple) ? 4 : 8));
LLVM_DECLARE_ATTRIBUTEREF(align_pool_attr, align, ponyint_pool_size(0));
LLVM_DECLARE_ATTRIBUTEREF(align_heap_attr, align, HEAP_MIN);
LLVM_DECLARE_ATTRIBUTEREF(deref_or_null_alloc_attr, dereferenceable_or_null,
HEAP_MIN);
LLVM_DECLARE_ATTRIBUTEREF(deref_alloc_small_attr, dereferenceable, HEAP_MIN);
LLVM_DECLARE_ATTRIBUTEREF(deref_alloc_large_attr, dereferenceable,
HEAP_MAX << 1);
#endif
// i8* pony_ctx()
type = LLVMFunctionType(c->void_ptr, NULL, 0, false);
value = LLVMAddFunction(c->module, "pony_ctx", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, readnone_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMAddFunctionAttr(value, LLVMReadNoneAttribute);
#endif
// __object* pony_create(i8*, __Desc*)
params[0] = c->void_ptr;
params[1] = c->descriptor_ptr;
type = LLVMFunctionType(c->object_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_create", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, noalias_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, deref_actor_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, align_pool_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
LLVMSetReturnNoAlias(value);
LLVMSetDereferenceable(value, 0, PONY_ACTOR_PAD_SIZE +
(target_is_ilp32(c->opt->triple) ? 4 : 8));
#endif
// void ponyint_destroy(__object*)
params[0] = c->object_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "ponyint_destroy", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
#endif
// void pony_sendv(i8*, __object*, $message*, $message*)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
params[2] = c->msg_ptr;
params[3] = c->msg_ptr;
type = LLVMFunctionType(c->void_type, params, 4, false);
value = LLVMAddFunction(c->module, "pony_sendv", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
#endif
// void pony_sendv_single(i8*, __object*, $message*, $message*)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
params[2] = c->msg_ptr;
params[3] = c->msg_ptr;
type = LLVMFunctionType(c->void_type, params, 4, false);
value = LLVMAddFunction(c->module, "pony_sendv_single", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
#endif
// i8* pony_alloc(i8*, intptr)
params[0] = c->void_ptr;
params[1] = c->intptr;
type = LLVMFunctionType(c->void_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, noalias_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex,
deref_or_null_alloc_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, align_heap_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
LLVMSetReturnNoAlias(value);
LLVMSetDereferenceableOrNull(value, 0, HEAP_MIN);
#endif
// i8* pony_alloc_small(i8*, i32)
params[0] = c->void_ptr;
params[1] = c->i32;
type = LLVMFunctionType(c->void_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc_small", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, noalias_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex,
deref_alloc_small_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, align_heap_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
LLVMSetReturnNoAlias(value);
LLVMSetDereferenceable(value, 0, HEAP_MIN);
#endif
// i8* pony_alloc_large(i8*, intptr)
params[0] = c->void_ptr;
params[1] = c->intptr;
type = LLVMFunctionType(c->void_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc_large", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, noalias_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex,
deref_alloc_large_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, align_heap_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
LLVMSetReturnNoAlias(value);
LLVMSetDereferenceable(value, 0, HEAP_MAX << 1);
#endif
// i8* pony_realloc(i8*, i8*, intptr)
params[0] = c->void_ptr;
params[1] = c->void_ptr;
params[2] = c->intptr;
type = LLVMFunctionType(c->void_ptr, params, 3, false);
value = LLVMAddFunction(c->module, "pony_realloc", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, noalias_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex,
deref_or_null_alloc_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, align_heap_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
LLVMSetReturnNoAlias(value);
LLVMSetDereferenceableOrNull(value, 0, HEAP_MIN);
#endif
// i8* pony_alloc_final(i8*, intptr)
params[0] = c->void_ptr;
params[1] = c->intptr;
type = LLVMFunctionType(c->void_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc_final", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, noalias_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex,
deref_or_null_alloc_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, align_heap_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
LLVMSetReturnNoAlias(value);
LLVMSetDereferenceableOrNull(value, 0, HEAP_MIN);
#endif
// i8* pony_alloc_small_final(i8*, i32)
params[0] = c->void_ptr;
params[1] = c->i32;
type = LLVMFunctionType(c->void_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc_small_final", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, noalias_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex,
deref_alloc_small_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, align_heap_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
LLVMSetReturnNoAlias(value);
LLVMSetDereferenceable(value, 0, HEAP_MIN);
#endif
// i8* pony_alloc_large_final(i8*, intptr)
params[0] = c->void_ptr;
params[1] = c->intptr;
type = LLVMFunctionType(c->void_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc_large_final", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, noalias_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex,
deref_alloc_large_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, align_heap_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
LLVMSetReturnNoAlias(value);
LLVMSetDereferenceable(value, 0, HEAP_MAX << 1);
#endif
// $message* pony_alloc_msg(i32, i32)
params[0] = c->i32;
params[1] = c->i32;
type = LLVMFunctionType(c->msg_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc_msg", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, noalias_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, align_pool_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
LLVMSetReturnNoAlias(value);
#endif
// void pony_trace(i8*, i8*)
params[0] = c->void_ptr;
params[1] = c->void_ptr;
type = LLVMFunctionType(c->void_type, params, 2, false);
value = LLVMAddFunction(c->module, "pony_trace", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, 2, readnone_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
value = LLVMGetParam(value, 1);
LLVMAddAttribute(value, LLVMReadNoneAttribute);
#endif
// void pony_traceknown(i8*, __object*, __Desc*, i32)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
params[2] = c->descriptor_ptr;
params[3] = c->i32;
type = LLVMFunctionType(c->void_type, params, 4, false);
value = LLVMAddFunction(c->module, "pony_traceknown", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, 2, readonly_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
value = LLVMGetParam(value, 1);
LLVMAddAttribute(value, LLVMReadOnlyAttribute);
#endif
// void pony_traceunknown(i8*, __object*, i32)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
params[2] = c->i32;
type = LLVMFunctionType(c->void_type, params, 3, false);
value = LLVMAddFunction(c->module, "pony_traceunknown", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, 2, readonly_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
value = LLVMGetParam(value, 1);
LLVMAddAttribute(value, LLVMReadOnlyAttribute);
#endif
// void pony_gc_send(i8*)
params[0] = c->void_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_gc_send", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
#endif
// void pony_gc_recv(i8*)
params[0] = c->void_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_gc_recv", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
#endif
// void pony_send_done(i8*)
params[0] = c->void_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_send_done", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
#endif
// void pony_recv_done(i8*)
params[0] = c->void_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_recv_done", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
#endif
// void pony_serialise_reserve(i8*, i8*, intptr)
params[0] = c->void_ptr;
params[1] = c->void_ptr;
params[2] = c->intptr;
type = LLVMFunctionType(c->void_type, params, 3, false);
value = LLVMAddFunction(c->module, "pony_serialise_reserve", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, 2, readnone_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
value = LLVMGetParam(value, 1);
LLVMAddAttribute(value, LLVMReadNoneAttribute);
#endif
// intptr pony_serialise_offset(i8*, i8*)
params[0] = c->void_ptr;
params[1] = c->void_ptr;
type = LLVMFunctionType(c->intptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_serialise_offset", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, 2, readonly_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
value = LLVMGetParam(value, 1);
LLVMAddAttribute(value, LLVMReadOnlyAttribute);
#endif
// i8* pony_deserialise_offset(i8*, __desc*, intptr)
params[0] = c->void_ptr;
params[1] = c->descriptor_ptr;
params[2] = c->intptr;
type = LLVMFunctionType(c->void_ptr, params, 3, false);
value = LLVMAddFunction(c->module, "pony_deserialise_offset", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
#elif PONY_LLVM == 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
#endif
// i8* pony_deserialise_block(i8*, intptr, intptr)
params[0] = c->void_ptr;
params[1] = c->intptr;
params[2] = c->intptr;
type = LLVMFunctionType(c->void_ptr, params, 3, false);
value = LLVMAddFunction(c->module, "pony_deserialise_block", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeReturnIndex, noalias_attr);
#else
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
LLVMSetReturnNoAlias(value);
#endif
// i32 pony_init(i32, i8**)
params[0] = c->i32;
params[1] = LLVMPointerType(c->void_ptr, 0);
type = LLVMFunctionType(c->i32, params, 2, false);
value = LLVMAddFunction(c->module, "pony_init", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
#endif
// void pony_become(i8*, __object*)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
type = LLVMFunctionType(c->void_type, params, 2, false);
value = LLVMAddFunction(c->module, "pony_become", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
#endif
// i32 pony_start(i32, i32)
params[0] = c->i32;
params[1] = c->i32;
type = LLVMFunctionType(c->i32, params, 2, false);
value = LLVMAddFunction(c->module, "pony_start", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex,
inacc_or_arg_mem_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
# if PONY_LLVM >= 308
LLVMSetInaccessibleMemOrArgMemOnly(value);
# endif
#endif
// i32 pony_get_exitcode()
type = LLVMFunctionType(c->i32, NULL, 0, false);
value = LLVMAddFunction(c->module, "pony_get_exitcode", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, readonly_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMAddFunctionAttr(value, LLVMReadOnlyAttribute);
#endif
// void pony_throw()
type = LLVMFunctionType(c->void_type, NULL, 0, false);
value = LLVMAddFunction(c->module, "pony_throw", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, noreturn_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoReturnAttribute);
#endif
// i32 pony_personality_v0(...)
type = LLVMFunctionType(c->i32, NULL, 0, true);
c->personality = LLVMAddFunction(c->module, "pony_personality_v0", type);
// i32 memcmp(i8*, i8*, intptr)
params[0] = c->void_ptr;
params[1] = c->void_ptr;
params[2] = c->intptr;
type = LLVMFunctionType(c->i32, params, 3, false);
value = LLVMAddFunction(c->module, "memcmp", type);
#if PONY_LLVM >= 309
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, nounwind_attr);
LLVMAddAttributeAtIndex(value, LLVMAttributeFunctionIndex, readonly_attr);
LLVMAddAttributeAtIndex(value, 1, readonly_attr);
LLVMAddAttributeAtIndex(value, 2, readonly_attr);
#else
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMAddFunctionAttr(value, LLVMReadOnlyAttribute);
LLVMValueRef param = LLVMGetParam(value, 0);
LLVMAddAttribute(param, LLVMReadOnlyAttribute);
param = LLVMGetParam(value, 1);
LLVMAddAttribute(param, LLVMReadOnlyAttribute);
#endif
}
static void declare_input_fs(
struct si_shader_context * si_shader_ctx,
unsigned input_index,
const struct tgsi_full_declaration *decl)
{
struct si_shader *shader = &si_shader_ctx->shader->shader;
struct lp_build_context * base =
&si_shader_ctx->radeon_bld.soa.bld_base.base;
struct gallivm_state * gallivm = base->gallivm;
LLVMTypeRef input_type = LLVMFloatTypeInContext(gallivm->context);
LLVMValueRef main_fn = si_shader_ctx->radeon_bld.main_fn;
LLVMValueRef interp_param;
const char * intr_name;
/* This value is:
* [15:0] NewPrimMask (Bit mask for each quad. It is set it the
* quad begins a new primitive. Bit 0 always needs
* to be unset)
* [32:16] ParamOffset
*
*/
LLVMValueRef params = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_PRIM_MASK);
LLVMValueRef attr_number;
unsigned chan;
if (decl->Semantic.Name == TGSI_SEMANTIC_POSITION) {
for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
unsigned soa_index =
radeon_llvm_reg_index_soa(input_index, chan);
si_shader_ctx->radeon_bld.inputs[soa_index] =
LLVMGetParam(main_fn, SI_PARAM_POS_X_FLOAT + chan);
if (chan == 3)
/* RCP for fragcoord.w */
si_shader_ctx->radeon_bld.inputs[soa_index] =
LLVMBuildFDiv(gallivm->builder,
lp_build_const_float(gallivm, 1.0f),
si_shader_ctx->radeon_bld.inputs[soa_index],
"");
}
return;
}
if (decl->Semantic.Name == TGSI_SEMANTIC_FACE) {
LLVMValueRef face, is_face_positive;
face = LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE);
is_face_positive = LLVMBuildFCmp(gallivm->builder,
LLVMRealUGT, face,
lp_build_const_float(gallivm, 0.0f),
"");
si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, 0)] =
LLVMBuildSelect(gallivm->builder,
is_face_positive,
lp_build_const_float(gallivm, 1.0f),
lp_build_const_float(gallivm, 0.0f),
"");
si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, 1)] =
si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, 2)] =
lp_build_const_float(gallivm, 0.0f);
si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, 3)] =
lp_build_const_float(gallivm, 1.0f);
return;
}
shader->input[input_index].param_offset = shader->ninterp++;
attr_number = lp_build_const_int32(gallivm,
shader->input[input_index].param_offset);
/* XXX: Handle all possible interpolation modes */
switch (decl->Interp.Interpolate) {
case TGSI_INTERPOLATE_COLOR:
if (si_shader_ctx->shader->key.ps.flatshade) {
interp_param = 0;
} else {
if (decl->Interp.Centroid)
interp_param = LLVMGetParam(main_fn, SI_PARAM_PERSP_CENTROID);
else
interp_param = LLVMGetParam(main_fn, SI_PARAM_PERSP_CENTER);
}
break;
case TGSI_INTERPOLATE_CONSTANT:
interp_param = 0;
break;
case TGSI_INTERPOLATE_LINEAR:
if (decl->Interp.Centroid)
interp_param = LLVMGetParam(main_fn, SI_PARAM_LINEAR_CENTROID);
else
interp_param = LLVMGetParam(main_fn, SI_PARAM_LINEAR_CENTER);
break;
case TGSI_INTERPOLATE_PERSPECTIVE:
if (decl->Interp.Centroid)
interp_param = LLVMGetParam(main_fn, SI_PARAM_PERSP_CENTROID);
else
interp_param = LLVMGetParam(main_fn, SI_PARAM_PERSP_CENTER);
break;
default:
fprintf(stderr, "Warning: Unhandled interpolation mode.\n");
return;
}
intr_name = interp_param ? "llvm.SI.fs.interp" : "llvm.SI.fs.constant";
/* XXX: Could there be more than TGSI_NUM_CHANNELS (4) ? */
if (decl->Semantic.Name == TGSI_SEMANTIC_COLOR &&
si_shader_ctx->shader->key.ps.color_two_side) {
LLVMValueRef args[4];
LLVMValueRef face, is_face_positive;
LLVMValueRef back_attr_number =
lp_build_const_int32(gallivm,
shader->input[input_index].param_offset + 1);
face = LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE);
is_face_positive = LLVMBuildFCmp(gallivm->builder,
LLVMRealUGT, face,
lp_build_const_float(gallivm, 0.0f),
"");
args[2] = params;
args[3] = interp_param;
for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
unsigned soa_index = radeon_llvm_reg_index_soa(input_index, chan);
LLVMValueRef front, back;
args[0] = llvm_chan;
args[1] = attr_number;
front = build_intrinsic(base->gallivm->builder, intr_name,
input_type, args, args[3] ? 4 : 3,
LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
args[1] = back_attr_number;
back = build_intrinsic(base->gallivm->builder, intr_name,
input_type, args, args[3] ? 4 : 3,
LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
si_shader_ctx->radeon_bld.inputs[soa_index] =
LLVMBuildSelect(gallivm->builder,
is_face_positive,
front,
back,
"");
}
shader->ninterp++;
} else {
for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
LLVMValueRef args[4];
LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
unsigned soa_index = radeon_llvm_reg_index_soa(input_index, chan);
args[0] = llvm_chan;
args[1] = attr_number;
args[2] = params;
args[3] = interp_param;
si_shader_ctx->radeon_bld.inputs[soa_index] =
build_intrinsic(base->gallivm->builder, intr_name,
input_type, args, args[3] ? 4 : 3,
LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
}
}
}
/**
* Return mask ? a : b;
*
* mask is a bitwise mask, composed of 0 or ~0 for each element. Any other value
* will yield unpredictable results.
*/
LLVMValueRef
lp_build_select(struct lp_build_context *bld,
LLVMValueRef mask,
LLVMValueRef a,
LLVMValueRef b)
{
LLVMBuilderRef builder = bld->gallivm->builder;
LLVMContextRef lc = bld->gallivm->context;
struct lp_type type = bld->type;
LLVMValueRef res;
assert(lp_check_value(type, a));
assert(lp_check_value(type, b));
if(a == b)
return a;
if (type.length == 1) {
mask = LLVMBuildTrunc(builder, mask, LLVMInt1TypeInContext(lc), "");
res = LLVMBuildSelect(builder, mask, a, b, "");
}
else if (0) {
/* Generate a vector select.
*
* XXX: Using vector selects would avoid emitting intrinsics, but they aren't
* properly supported yet.
*
* LLVM 3.0 includes experimental support provided the -promote-elements
* options is passed to LLVM's command line (e.g., via
* llvm::cl::ParseCommandLineOptions), but resulting code quality is much
* worse, probably because some optimization passes don't know how to
* handle vector selects.
*
* See also:
* - http://lists.cs.uiuc.edu/pipermail/llvmdev/2011-October/043659.html
*/
/* Convert the mask to a vector of booleans.
* XXX: There are two ways to do this. Decide what's best.
*/
if (1) {
LLVMTypeRef bool_vec_type = LLVMVectorType(LLVMInt1TypeInContext(lc), type.length);
mask = LLVMBuildTrunc(builder, mask, bool_vec_type, "");
} else {
mask = LLVMBuildICmp(builder, LLVMIntNE, mask, LLVMConstNull(bld->int_vec_type), "");
}
res = LLVMBuildSelect(builder, mask, a, b, "");
}
else if (((util_cpu_caps.has_sse4_1 &&
type.width * type.length == 128) ||
(util_cpu_caps.has_avx &&
type.width * type.length == 256 && type.width >= 32)) &&
!LLVMIsConstant(a) &&
!LLVMIsConstant(b) &&
!LLVMIsConstant(mask)) {
const char *intrinsic;
LLVMTypeRef arg_type;
LLVMValueRef args[3];
/*
* There's only float blend in AVX but can just cast i32/i64
* to float.
*/
if (type.width * type.length == 256) {
if (type.width == 64) {
intrinsic = "llvm.x86.avx.blendv.pd.256";
arg_type = LLVMVectorType(LLVMDoubleTypeInContext(lc), 4);
}
else {
intrinsic = "llvm.x86.avx.blendv.ps.256";
arg_type = LLVMVectorType(LLVMFloatTypeInContext(lc), 8);
}
}
else if (type.floating &&
type.width == 64) {
intrinsic = "llvm.x86.sse41.blendvpd";
arg_type = LLVMVectorType(LLVMDoubleTypeInContext(lc), 2);
} else if (type.floating &&
type.width == 32) {
intrinsic = "llvm.x86.sse41.blendvps";
arg_type = LLVMVectorType(LLVMFloatTypeInContext(lc), 4);
} else {
intrinsic = "llvm.x86.sse41.pblendvb";
arg_type = LLVMVectorType(LLVMInt8TypeInContext(lc), 16);
}
if (arg_type != bld->int_vec_type) {
mask = LLVMBuildBitCast(builder, mask, arg_type, "");
}
if (arg_type != bld->vec_type) {
a = LLVMBuildBitCast(builder, a, arg_type, "");
b = LLVMBuildBitCast(builder, b, arg_type, "");
}
args[0] = b;
args[1] = a;
args[2] = mask;
res = lp_build_intrinsic(builder, intrinsic,
arg_type, args, Elements(args));
if (arg_type != bld->vec_type) {
res = LLVMBuildBitCast(builder, res, bld->vec_type, "");
}
}
else {
res = lp_build_select_bitwise(bld, mask, a, b);
}
return res;
}
static LLVMValueRef
lp_build_gather_avx2(struct gallivm_state *gallivm,
unsigned length,
unsigned src_width,
struct lp_type dst_type,
LLVMValueRef base_ptr,
LLVMValueRef offsets)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMTypeRef src_type, src_vec_type;
LLVMValueRef res;
struct lp_type res_type = dst_type;
res_type.length *= length;
if (dst_type.floating) {
src_type = src_width == 64 ? LLVMDoubleTypeInContext(gallivm->context) :
LLVMFloatTypeInContext(gallivm->context);
} else {
src_type = LLVMIntTypeInContext(gallivm->context, src_width);
}
src_vec_type = LLVMVectorType(src_type, length);
/* XXX should allow hw scaling (can handle i8, i16, i32, i64 for x86) */
assert(LLVMTypeOf(base_ptr) == LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0));
if (0) {
/*
* XXX: This will cause LLVM pre 3.7 to hang; it works on LLVM 3.8 but
* will not use the AVX2 gather instrinsics (even with llvm 4.0), at
* least with Haswell. See
* http://lists.llvm.org/pipermail/llvm-dev/2016-January/094448.html
* And the generated code doing the emulation is quite a bit worse
* than what we get by doing it ourselves too.
*/
LLVMTypeRef i32_type = LLVMIntTypeInContext(gallivm->context, 32);
LLVMTypeRef i32_vec_type = LLVMVectorType(i32_type, length);
LLVMTypeRef i1_type = LLVMIntTypeInContext(gallivm->context, 1);
LLVMTypeRef i1_vec_type = LLVMVectorType(i1_type, length);
LLVMTypeRef src_ptr_type = LLVMPointerType(src_type, 0);
LLVMValueRef src_ptr;
base_ptr = LLVMBuildBitCast(builder, base_ptr, src_ptr_type, "");
/* Rescale offsets from bytes to elements */
LLVMValueRef scale = LLVMConstInt(i32_type, src_width/8, 0);
scale = lp_build_broadcast(gallivm, i32_vec_type, scale);
assert(LLVMTypeOf(offsets) == i32_vec_type);
offsets = LLVMBuildSDiv(builder, offsets, scale, "");
src_ptr = LLVMBuildGEP(builder, base_ptr, &offsets, 1, "vector-gep");
char intrinsic[64];
util_snprintf(intrinsic, sizeof intrinsic, "llvm.masked.gather.v%u%s%u",
length, dst_type.floating ? "f" : "i", src_width);
LLVMValueRef alignment = LLVMConstInt(i32_type, src_width/8, 0);
LLVMValueRef mask = LLVMConstAllOnes(i1_vec_type);
LLVMValueRef passthru = LLVMGetUndef(src_vec_type);
LLVMValueRef args[] = { src_ptr, alignment, mask, passthru };
res = lp_build_intrinsic(builder, intrinsic, src_vec_type, args, 4, 0);
} else {
LLVMTypeRef i8_type = LLVMIntTypeInContext(gallivm->context, 8);
const char *intrinsic = NULL;
unsigned l_idx = 0;
assert(src_width == 32 || src_width == 64);
if (src_width == 32) {
assert(length == 4 || length == 8);
} else {
assert(length == 2 || length == 4);
}
static const char *intrinsics[2][2][2] = {
{{"llvm.x86.avx2.gather.d.d",
"llvm.x86.avx2.gather.d.d.256"},
{"llvm.x86.avx2.gather.d.q",
"llvm.x86.avx2.gather.d.q.256"}},
{{"llvm.x86.avx2.gather.d.ps",
"llvm.x86.avx2.gather.d.ps.256"},
{"llvm.x86.avx2.gather.d.pd",
"llvm.x86.avx2.gather.d.pd.256"}},
};
if ((src_width == 32 && length == 8) ||
(src_width == 64 && length == 4)) {
l_idx = 1;
}
intrinsic = intrinsics[dst_type.floating][src_width == 64][l_idx];
LLVMValueRef passthru = LLVMGetUndef(src_vec_type);
LLVMValueRef mask = LLVMConstAllOnes(src_vec_type);
mask = LLVMConstBitCast(mask, src_vec_type);
LLVMValueRef scale = LLVMConstInt(i8_type, 1, 0);
LLVMValueRef args[] = { passthru, base_ptr, offsets, mask, scale };
res = lp_build_intrinsic(builder, intrinsic, src_vec_type, args, 5, 0);
}
res = LLVMBuildBitCast(builder, res, lp_build_vec_type(gallivm, res_type), "");
return res;
}
/**
* Converts float32 to int16 half-float
* Note this can be performed in 1 instruction if vcvtps2ph exists (f16c/cvt16)
* [llvm.x86.vcvtps2ph / _mm_cvtps_ph]
*
* @param src value to convert
*
* Convert float32 to half floats, preserving Infs and NaNs,
* with rounding towards zero (trunc).
*/
LLVMValueRef
lp_build_float_to_half(struct gallivm_state *gallivm,
LLVMValueRef src)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMTypeRef f32_vec_type = LLVMTypeOf(src);
unsigned length = LLVMGetTypeKind(f32_vec_type) == LLVMVectorTypeKind
? LLVMGetVectorSize(f32_vec_type) : 1;
struct lp_type i32_type = lp_type_int_vec(32, 32 * length);
struct lp_type i16_type = lp_type_int_vec(16, 16 * length);
LLVMValueRef result;
if (util_cpu_caps.has_f16c && HAVE_LLVM >= 0x0301 &&
(length == 4 || length == 8)) {
struct lp_type i168_type = lp_type_int_vec(16, 16 * 8);
unsigned mode = 3; /* same as LP_BUILD_ROUND_TRUNCATE */
LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
const char *intrinsic = NULL;
if (length == 4) {
intrinsic = "llvm.x86.vcvtps2ph.128";
}
else {
intrinsic = "llvm.x86.vcvtps2ph.256";
}
result = lp_build_intrinsic_binary(builder, intrinsic,
lp_build_vec_type(gallivm, i168_type),
src, LLVMConstInt(i32t, mode, 0));
if (length == 4) {
result = lp_build_extract_range(gallivm, result, 0, 4);
}
}
else {
result = lp_build_float_to_smallfloat(gallivm, i32_type, src, 10, 5, 0, true);
/* Convert int32 vector to int16 vector by trunc (might generate bad code) */
result = LLVMBuildTrunc(builder, result, lp_build_vec_type(gallivm, i16_type), "");
}
/*
* Debugging code.
*/
if (0) {
LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
LLVMTypeRef i16t = LLVMInt16TypeInContext(gallivm->context);
LLVMTypeRef f32t = LLVMFloatTypeInContext(gallivm->context);
LLVMValueRef ref_result = LLVMGetUndef(LLVMVectorType(i16t, length));
unsigned i;
LLVMTypeRef func_type = LLVMFunctionType(i16t, &f32t, 1, 0);
LLVMValueRef func = lp_build_const_int_pointer(gallivm, func_to_pointer((func_pointer)util_float_to_half));
func = LLVMBuildBitCast(builder, func, LLVMPointerType(func_type, 0), "util_float_to_half");
for (i = 0; i < length; ++i) {
LLVMValueRef index = LLVMConstInt(i32t, i, 0);
LLVMValueRef f32 = LLVMBuildExtractElement(builder, src, index, "");
#if 0
/* XXX: not really supported by backends */
LLVMValueRef f16 = lp_build_intrinsic_unary(builder, "llvm.convert.to.fp16", i16t, f32);
#else
LLVMValueRef f16 = LLVMBuildCall(builder, func, &f32, 1, "");
#endif
ref_result = LLVMBuildInsertElement(builder, ref_result, f16, index, "");
}
lp_build_print_value(gallivm, "src = ", src);
lp_build_print_value(gallivm, "llvm = ", result);
lp_build_print_value(gallivm, "util = ", ref_result);
lp_build_printf(gallivm, "\n");
}
return result;
}
static void init_runtime(compile_t* c)
{
c->str_builtin = stringtab("$0");
c->str_Bool = stringtab("Bool");
c->str_I8 = stringtab("I8");
c->str_I16 = stringtab("I16");
c->str_I32 = stringtab("I32");
c->str_I64 = stringtab("I64");
c->str_I128 = stringtab("I128");
c->str_ILong = stringtab("ILong");
c->str_ISize = stringtab("ISize");
c->str_U8 = stringtab("U8");
c->str_U16 = stringtab("U16");
c->str_U32 = stringtab("U32");
c->str_U64 = stringtab("U64");
c->str_U128 = stringtab("U128");
c->str_ULong = stringtab("ULong");
c->str_USize = stringtab("USize");
c->str_F32 = stringtab("F32");
c->str_F64 = stringtab("F64");
c->str_Pointer = stringtab("Pointer");
c->str_Maybe = stringtab("MaybePointer");
c->str_DoNotOptimise = stringtab("DoNotOptimise");
c->str_Array = stringtab("Array");
c->str_String = stringtab("String");
c->str_Platform = stringtab("Platform");
c->str_Main = stringtab("Main");
c->str_Env = stringtab("Env");
c->str_add = stringtab("add");
c->str_sub = stringtab("sub");
c->str_mul = stringtab("mul");
c->str_div = stringtab("div");
c->str_mod = stringtab("mod");
c->str_neg = stringtab("neg");
c->str_and = stringtab("op_and");
c->str_or = stringtab("op_or");
c->str_xor = stringtab("op_xor");
c->str_not = stringtab("op_not");
c->str_shl = stringtab("shl");
c->str_shr = stringtab("shr");
c->str_eq = stringtab("eq");
c->str_ne = stringtab("ne");
c->str_lt = stringtab("lt");
c->str_le = stringtab("le");
c->str_ge = stringtab("ge");
c->str_gt = stringtab("gt");
c->str_this = stringtab("this");
c->str_create = stringtab("create");
c->str__create = stringtab("_create");
c->str__init = stringtab("_init");
c->str__final = stringtab("_final");
c->str__event_notify = stringtab("_event_notify");
LLVMTypeRef type;
LLVMTypeRef params[5];
LLVMValueRef value;
c->void_type = LLVMVoidTypeInContext(c->context);
c->ibool = LLVMInt8TypeInContext(c->context);
c->i1 = LLVMInt1TypeInContext(c->context);
c->i8 = LLVMInt8TypeInContext(c->context);
c->i16 = LLVMInt16TypeInContext(c->context);
c->i32 = LLVMInt32TypeInContext(c->context);
c->i64 = LLVMInt64TypeInContext(c->context);
c->i128 = LLVMIntTypeInContext(c->context, 128);
c->f32 = LLVMFloatTypeInContext(c->context);
c->f64 = LLVMDoubleTypeInContext(c->context);
c->intptr = LLVMIntPtrTypeInContext(c->context, c->target_data);
// i8*
c->void_ptr = LLVMPointerType(c->i8, 0);
// forward declare object
c->object_type = LLVMStructCreateNamed(c->context, "__object");
c->object_ptr = LLVMPointerType(c->object_type, 0);
// padding required in an actor between the descriptor and fields
c->actor_pad = LLVMArrayType(c->i8, PONY_ACTOR_PAD_SIZE);
// message
params[0] = c->i32; // size
params[1] = c->i32; // id
c->msg_type = LLVMStructCreateNamed(c->context, "__message");
c->msg_ptr = LLVMPointerType(c->msg_type, 0);
LLVMStructSetBody(c->msg_type, params, 2, false);
// trace
// void (*)(i8*, __object*)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
c->trace_type = LLVMFunctionType(c->void_type, params, 2, false);
c->trace_fn = LLVMPointerType(c->trace_type, 0);
// serialise
// void (*)(i8*, __object*, i8*, intptr, i32)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
params[2] = c->void_ptr;
params[3] = c->intptr;
params[4] = c->i32;
c->serialise_type = LLVMFunctionType(c->void_type, params, 5, false);
c->serialise_fn = LLVMPointerType(c->serialise_type, 0);
// dispatch
// void (*)(i8*, __object*, $message*)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
params[2] = c->msg_ptr;
c->dispatch_type = LLVMFunctionType(c->void_type, params, 3, false);
c->dispatch_fn = LLVMPointerType(c->dispatch_type, 0);
// void (*)(__object*)
params[0] = c->object_ptr;
c->final_fn = LLVMPointerType(
LLVMFunctionType(c->void_type, params, 1, false), 0);
// descriptor, opaque version
// We need this in order to build our own structure.
const char* desc_name = genname_descriptor(NULL);
c->descriptor_type = LLVMStructCreateNamed(c->context, desc_name);
c->descriptor_ptr = LLVMPointerType(c->descriptor_type, 0);
// field descriptor
// Also needed to build a descriptor structure.
params[0] = c->i32;
params[1] = c->descriptor_ptr;
c->field_descriptor = LLVMStructTypeInContext(c->context, params, 2, false);
// descriptor, filled in
gendesc_basetype(c, c->descriptor_type);
// define object
params[0] = c->descriptor_ptr;
LLVMStructSetBody(c->object_type, params, 1, false);
// $i8* pony_ctx()
type = LLVMFunctionType(c->void_ptr, NULL, 0, false);
value = LLVMAddFunction(c->module, "pony_ctx", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMAddFunctionAttr(value, LLVMReadNoneAttribute);
// __object* pony_create(i8*, __Desc*)
params[0] = c->void_ptr;
params[1] = c->descriptor_ptr;
type = LLVMFunctionType(c->object_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_create", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMSetReturnNoAlias(value);
LLVMSetDereferenceable(value, 0, PONY_ACTOR_PAD_SIZE);
// void ponyint_destroy(__object*)
params[0] = c->object_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "ponyint_destroy", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_sendv(i8*, __object*, $message*);
params[0] = c->void_ptr;
params[1] = c->object_ptr;
params[2] = c->msg_ptr;
type = LLVMFunctionType(c->void_type, params, 3, false);
value = LLVMAddFunction(c->module, "pony_sendv", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// i8* pony_alloc(i8*, intptr)
params[0] = c->void_ptr;
params[1] = c->intptr;
type = LLVMFunctionType(c->void_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMSetReturnNoAlias(value);
#if PONY_LLVM >= 307
LLVMSetDereferenceableOrNull(value, 0, HEAP_MIN);
#endif
// i8* pony_alloc_small(i8*, i32)
params[0] = c->void_ptr;
params[1] = c->i32;
type = LLVMFunctionType(c->void_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc_small", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMSetReturnNoAlias(value);
LLVMSetDereferenceable(value, 0, HEAP_MIN);
// i8* pony_alloc_large(i8*, intptr)
params[0] = c->void_ptr;
params[1] = c->intptr;
type = LLVMFunctionType(c->void_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc_large", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMSetReturnNoAlias(value);
LLVMSetDereferenceable(value, 0, HEAP_MAX + 1);
// i8* pony_realloc(i8*, i8*, intptr)
params[0] = c->void_ptr;
params[1] = c->void_ptr;
params[2] = c->intptr;
type = LLVMFunctionType(c->void_ptr, params, 3, false);
value = LLVMAddFunction(c->module, "pony_realloc", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMSetReturnNoAlias(value);
#if PONY_LLVM >= 307
LLVMSetDereferenceableOrNull(value, 0, HEAP_MIN);
#endif
// i8* pony_alloc_final(i8*, intptr, c->final_fn)
params[0] = c->void_ptr;
params[1] = c->intptr;
params[2] = c->final_fn;
type = LLVMFunctionType(c->void_ptr, params, 3, false);
value = LLVMAddFunction(c->module, "pony_alloc_final", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMSetReturnNoAlias(value);
#if PONY_LLVM >= 307
LLVMSetDereferenceableOrNull(value, 0, HEAP_MIN);
#endif
// $message* pony_alloc_msg(i32, i32)
params[0] = c->i32;
params[1] = c->i32;
type = LLVMFunctionType(c->msg_ptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_alloc_msg", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
LLVMSetReturnNoAlias(value);
// void pony_trace(i8*, i8*)
params[0] = c->void_ptr;
params[1] = c->void_ptr;
type = LLVMFunctionType(c->void_type, params, 2, false);
value = LLVMAddFunction(c->module, "pony_trace", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// i8* pony_traceobject(i8*, __object*, __Desc*, i32)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
params[2] = c->descriptor_ptr;
params[3] = c->i32;
type = LLVMFunctionType(c->void_ptr, params, 4, false);
value = LLVMAddFunction(c->module, "pony_traceknown", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// i8* pony_traceunknown(i8*, __object*, i32)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
params[2] = c->i32;
type = LLVMFunctionType(c->void_ptr, params, 3, false);
value = LLVMAddFunction(c->module, "pony_traceunknown", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_gc_send(i8*)
params[0] = c->void_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_gc_send", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_gc_recv(i8*)
params[0] = c->void_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_gc_recv", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_send_done(i8*)
params[0] = c->void_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_send_done", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_recv_done(i8*)
params[0] = c->void_ptr;
type = LLVMFunctionType(c->void_type, params, 1, false);
value = LLVMAddFunction(c->module, "pony_recv_done", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_serialise_reserve(i8*, i8*, intptr)
params[0] = c->void_ptr;
params[1] = c->void_ptr;
params[2] = c->intptr;
type = LLVMFunctionType(c->void_type, params, 3, false);
value = LLVMAddFunction(c->module, "pony_serialise_reserve", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// intptr pony_serialise_offset(i8*, i8*)
params[0] = c->void_ptr;
params[1] = c->void_ptr;
type = LLVMFunctionType(c->intptr, params, 2, false);
value = LLVMAddFunction(c->module, "pony_serialise_offset", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// i8* pony_deserialise_offset(i8*, __desc*, intptr)
params[0] = c->void_ptr;
params[1] = c->descriptor_ptr;
params[2] = c->intptr;
type = LLVMFunctionType(c->void_ptr, params, 3, false);
value = LLVMAddFunction(c->module, "pony_deserialise_offset", type);
// i8* pony_deserialise_block(i8*, intptr, intptr)
params[0] = c->void_ptr;
params[1] = c->intptr;
params[2] = c->intptr;
type = LLVMFunctionType(c->void_ptr, params, 3, false);
value = LLVMAddFunction(c->module, "pony_deserialise_block", type);
// i32 pony_init(i32, i8**)
params[0] = c->i32;
params[1] = LLVMPointerType(c->void_ptr, 0);
type = LLVMFunctionType(c->i32, params, 2, false);
value = LLVMAddFunction(c->module, "pony_init", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_become(i8*, __object*)
params[0] = c->void_ptr;
params[1] = c->object_ptr;
type = LLVMFunctionType(c->void_type, params, 2, false);
value = LLVMAddFunction(c->module, "pony_become", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// i32 pony_start(i32)
params[0] = c->i32;
type = LLVMFunctionType(c->i32, params, 1, false);
value = LLVMAddFunction(c->module, "pony_start", type);
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void pony_throw()
type = LLVMFunctionType(c->void_type, NULL, 0, false);
LLVMAddFunction(c->module, "pony_throw", type);
// i32 pony_personality_v0(...)
type = LLVMFunctionType(c->i32, NULL, 0, true);
c->personality = LLVMAddFunction(c->module, "pony_personality_v0", type);
// void llvm.memcpy.*(i8*, i8*, i32/64, i32, i1)
params[0] = c->void_ptr;
params[1] = c->void_ptr;
params[3] = c->i32;
params[4] = c->i1;
if(target_is_ilp32(c->opt->triple))
{
params[2] = c->i32;
type = LLVMFunctionType(c->void_type, params, 5, false);
value = LLVMAddFunction(c->module, "llvm.memcpy.p0i8.p0i8.i32", type);
} else {
params[2] = c->i64;
type = LLVMFunctionType(c->void_type, params, 5, false);
value = LLVMAddFunction(c->module, "llvm.memcpy.p0i8.p0i8.i64", type);
}
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
// void llvm.memmove.*(i8*, i8*, i32/64, i32, i1)
params[0] = c->void_ptr;
params[1] = c->void_ptr;
params[3] = c->i32;
params[4] = c->i1;
if(target_is_ilp32(c->opt->triple))
{
params[2] = c->i32;
type = LLVMFunctionType(c->void_type, params, 5, false);
value = LLVMAddFunction(c->module, "llvm.memmove.p0i8.p0i8.i32", type);
} else {
params[2] = c->i64;
type = LLVMFunctionType(c->void_type, params, 5, false);
value = LLVMAddFunction(c->module, "llvm.memmove.p0i8.p0i8.i64", type);
}
LLVMAddFunctionAttr(value, LLVMNoUnwindAttribute);
}
