bool
vtn_handle_glsl450_instruction(struct vtn_builder *b, uint32_t ext_opcode,
const uint32_t *w, unsigned count)
{
switch ((enum GLSLstd450)ext_opcode) {
case GLSLstd450Determinant: {
struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
val->ssa = rzalloc(b, struct vtn_ssa_value);
val->ssa->type = vtn_value(b, w[1], vtn_value_type_type)->type->type;
val->ssa->def = build_mat_det(b, vtn_ssa_value(b, w[5]));
break;
}
case GLSLstd450MatrixInverse: {
struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
val->ssa = matrix_inverse(b, vtn_ssa_value(b, w[5]));
break;
}
case GLSLstd450InterpolateAtCentroid:
case GLSLstd450InterpolateAtSample:
case GLSLstd450InterpolateAtOffset:
handle_glsl450_interpolation(b, ext_opcode, w, count);
break;
default:
handle_glsl450_alu(b, (enum GLSLstd450)ext_opcode, w, count);
}
return true;
}
static nir_ssa_def *
vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link,
unsigned stride)
{
assert(stride > 0);
if (link.mode == vtn_access_mode_literal) {
return nir_imm_int(&b->nb, link.id * stride);
} else if (stride == 1) {
return vtn_ssa_value(b, link.id)->def;
} else {
return nir_imul(&b->nb, vtn_ssa_value(b, link.id)->def,
nir_imm_int(&b->nb, stride));
}
}
static void
handle_glsl450_interpolation(struct vtn_builder *b, enum GLSLstd450 opcode,
const uint32_t *w, unsigned count)
{
const struct glsl_type *dest_type =
vtn_value(b, w[1], vtn_value_type_type)->type->type;
struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
val->ssa = vtn_create_ssa_value(b, dest_type);
nir_intrinsic_op op;
switch (opcode) {
case GLSLstd450InterpolateAtCentroid:
op = nir_intrinsic_interp_var_at_centroid;
break;
case GLSLstd450InterpolateAtSample:
op = nir_intrinsic_interp_var_at_sample;
break;
case GLSLstd450InterpolateAtOffset:
op = nir_intrinsic_interp_var_at_offset;
break;
default:
unreachable("Invalid opcode");
}
nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->nb.shader, op);
nir_deref_var *deref = vtn_nir_deref(b, w[5]);
intrin->variables[0] =
nir_deref_as_var(nir_copy_deref(intrin, &deref->deref));
switch (opcode) {
case GLSLstd450InterpolateAtCentroid:
break;
case GLSLstd450InterpolateAtSample:
case GLSLstd450InterpolateAtOffset:
intrin->src[0] = nir_src_for_ssa(vtn_ssa_value(b, w[6])->def);
break;
default:
unreachable("Invalid opcode");
}
intrin->num_components = glsl_get_vector_elements(dest_type);
nir_ssa_dest_init(&intrin->instr, &intrin->dest,
glsl_get_vector_elements(dest_type),
glsl_get_bit_size(dest_type), NULL);
val->ssa->def = &intrin->dest.ssa;
nir_builder_instr_insert(&b->nb, &intrin->instr);
}
void
vtn_handle_alu(struct vtn_builder *b, SpvOp opcode,
const uint32_t *w, unsigned count)
{
struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
const struct glsl_type *type =
vtn_value(b, w[1], vtn_value_type_type)->type->type;
vtn_foreach_decoration(b, val, handle_no_contraction, NULL);
/* Collect the various SSA sources */
const unsigned num_inputs = count - 3;
struct vtn_ssa_value *vtn_src[4] = { NULL, };
for (unsigned i = 0; i < num_inputs; i++)
vtn_src[i] = vtn_ssa_value(b, w[i + 3]);
if (glsl_type_is_matrix(vtn_src[0]->type) ||
(num_inputs >= 2 && glsl_type_is_matrix(vtn_src[1]->type))) {
vtn_handle_matrix_alu(b, opcode, val, vtn_src[0], vtn_src[1]);
b->nb.exact = false;
return;
}
val->ssa = vtn_create_ssa_value(b, type);
nir_ssa_def *src[4] = { NULL, };
for (unsigned i = 0; i < num_inputs; i++) {
assert(glsl_type_is_vector_or_scalar(vtn_src[i]->type));
src[i] = vtn_src[i]->def;
}
switch (opcode) {
case SpvOpAny:
if (src[0]->num_components == 1) {
val->ssa->def = nir_imov(&b->nb, src[0]);
} else {
nir_op op;
switch (src[0]->num_components) {
case 2: op = nir_op_bany_inequal2; break;
case 3: op = nir_op_bany_inequal3; break;
case 4: op = nir_op_bany_inequal4; break;
default: unreachable("invalid number of components");
}
val->ssa->def = nir_build_alu(&b->nb, op, src[0],
nir_imm_int(&b->nb, NIR_FALSE),
NULL, NULL);
}
break;
case SpvOpAll:
if (src[0]->num_components == 1) {
val->ssa->def = nir_imov(&b->nb, src[0]);
} else {
nir_op op;
switch (src[0]->num_components) {
case 2: op = nir_op_ball_iequal2; break;
case 3: op = nir_op_ball_iequal3; break;
case 4: op = nir_op_ball_iequal4; break;
default: unreachable("invalid number of components");
}
val->ssa->def = nir_build_alu(&b->nb, op, src[0],
nir_imm_int(&b->nb, NIR_TRUE),
NULL, NULL);
}
break;
case SpvOpOuterProduct: {
for (unsigned i = 0; i < src[1]->num_components; i++) {
val->ssa->elems[i]->def =
nir_fmul(&b->nb, src[0], nir_channel(&b->nb, src[1], i));
}
break;
}
case SpvOpDot:
val->ssa->def = nir_fdot(&b->nb, src[0], src[1]);
break;
case SpvOpIAddCarry:
assert(glsl_type_is_struct(val->ssa->type));
val->ssa->elems[0]->def = nir_iadd(&b->nb, src[0], src[1]);
val->ssa->elems[1]->def = nir_uadd_carry(&b->nb, src[0], src[1]);
break;
case SpvOpISubBorrow:
assert(glsl_type_is_struct(val->ssa->type));
val->ssa->elems[0]->def = nir_isub(&b->nb, src[0], src[1]);
val->ssa->elems[1]->def = nir_usub_borrow(&b->nb, src[0], src[1]);
break;
case SpvOpUMulExtended:
assert(glsl_type_is_struct(val->ssa->type));
val->ssa->elems[0]->def = nir_imul(&b->nb, src[0], src[1]);
val->ssa->elems[1]->def = nir_umul_high(&b->nb, src[0], src[1]);
break;
case SpvOpSMulExtended:
assert(glsl_type_is_struct(val->ssa->type));
val->ssa->elems[0]->def = nir_imul(&b->nb, src[0], src[1]);
val->ssa->elems[1]->def = nir_imul_high(&b->nb, src[0], src[1]);
break;
case SpvOpFwidth:
val->ssa->def = nir_fadd(&b->nb,
nir_fabs(&b->nb, nir_fddx(&b->nb, src[0])),
nir_fabs(&b->nb, nir_fddy(&b->nb, src[0])));
break;
case SpvOpFwidthFine:
val->ssa->def = nir_fadd(&b->nb,
nir_fabs(&b->nb, nir_fddx_fine(&b->nb, src[0])),
nir_fabs(&b->nb, nir_fddy_fine(&b->nb, src[0])));
break;
case SpvOpFwidthCoarse:
val->ssa->def = nir_fadd(&b->nb,
nir_fabs(&b->nb, nir_fddx_coarse(&b->nb, src[0])),
nir_fabs(&b->nb, nir_fddy_coarse(&b->nb, src[0])));
break;
case SpvOpVectorTimesScalar:
/* The builder will take care of splatting for us. */
val->ssa->def = nir_fmul(&b->nb, src[0], src[1]);
break;
case SpvOpIsNan:
val->ssa->def = nir_fne(&b->nb, src[0], src[0]);
break;
case SpvOpIsInf:
val->ssa->def = nir_feq(&b->nb, nir_fabs(&b->nb, src[0]),
nir_imm_float(&b->nb, INFINITY));
break;
case SpvOpFUnordEqual:
case SpvOpFUnordNotEqual:
case SpvOpFUnordLessThan:
case SpvOpFUnordGreaterThan:
case SpvOpFUnordLessThanEqual:
case SpvOpFUnordGreaterThanEqual: {
bool swap;
nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
nir_op op = vtn_nir_alu_op_for_spirv_opcode(opcode, &swap, src_alu_type, dst_alu_type);
if (swap) {
nir_ssa_def *tmp = src[0];
src[0] = src[1];
src[1] = tmp;
}
val->ssa->def =
nir_ior(&b->nb,
nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL),
nir_ior(&b->nb,
nir_fne(&b->nb, src[0], src[0]),
nir_fne(&b->nb, src[1], src[1])));
break;
}
case SpvOpFOrdEqual:
case SpvOpFOrdNotEqual:
case SpvOpFOrdLessThan:
case SpvOpFOrdGreaterThan:
case SpvOpFOrdLessThanEqual:
case SpvOpFOrdGreaterThanEqual: {
bool swap;
nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
nir_op op = vtn_nir_alu_op_for_spirv_opcode(opcode, &swap, src_alu_type, dst_alu_type);
if (swap) {
nir_ssa_def *tmp = src[0];
src[0] = src[1];
src[1] = tmp;
}
val->ssa->def =
nir_iand(&b->nb,
nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL),
nir_iand(&b->nb,
nir_feq(&b->nb, src[0], src[0]),
nir_feq(&b->nb, src[1], src[1])));
break;
}
default: {
bool swap;
nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
nir_op op = vtn_nir_alu_op_for_spirv_opcode(opcode, &swap, src_alu_type, dst_alu_type);
if (swap) {
nir_ssa_def *tmp = src[0];
src[0] = src[1];
src[1] = tmp;
}
val->ssa->def = nir_build_alu(&b->nb, op, src[0], src[1], src[2], src[3]);
break;
} /* default */
}
b->nb.exact = false;
}
static void
handle_glsl450_alu(struct vtn_builder *b, enum GLSLstd450 entrypoint,
const uint32_t *w, unsigned count)
{
struct nir_builder *nb = &b->nb;
const struct glsl_type *dest_type =
vtn_value(b, w[1], vtn_value_type_type)->type->type;
struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
val->ssa = vtn_create_ssa_value(b, dest_type);
/* Collect the various SSA sources */
unsigned num_inputs = count - 5;
nir_ssa_def *src[3] = { NULL, };
for (unsigned i = 0; i < num_inputs; i++)
src[i] = vtn_ssa_value(b, w[i + 5])->def;
switch (entrypoint) {
case GLSLstd450Radians:
val->ssa->def = nir_fmul(nb, src[0], nir_imm_float(nb, 0.01745329251));
return;
case GLSLstd450Degrees:
val->ssa->def = nir_fmul(nb, src[0], nir_imm_float(nb, 57.2957795131));
return;
case GLSLstd450Tan:
val->ssa->def = nir_fdiv(nb, nir_fsin(nb, src[0]),
nir_fcos(nb, src[0]));
return;
case GLSLstd450Modf: {
nir_ssa_def *sign = nir_fsign(nb, src[0]);
nir_ssa_def *abs = nir_fabs(nb, src[0]);
val->ssa->def = nir_fmul(nb, sign, nir_ffract(nb, abs));
nir_store_deref_var(nb, vtn_nir_deref(b, w[6]),
nir_fmul(nb, sign, nir_ffloor(nb, abs)), 0xf);
return;
}
case GLSLstd450ModfStruct: {
nir_ssa_def *sign = nir_fsign(nb, src[0]);
nir_ssa_def *abs = nir_fabs(nb, src[0]);
assert(glsl_type_is_struct(val->ssa->type));
val->ssa->elems[0]->def = nir_fmul(nb, sign, nir_ffract(nb, abs));
val->ssa->elems[1]->def = nir_fmul(nb, sign, nir_ffloor(nb, abs));
return;
}
case GLSLstd450Step:
val->ssa->def = nir_sge(nb, src[1], src[0]);
return;
case GLSLstd450Length:
val->ssa->def = build_length(nb, src[0]);
return;
case GLSLstd450Distance:
val->ssa->def = build_length(nb, nir_fsub(nb, src[0], src[1]));
return;
case GLSLstd450Normalize:
val->ssa->def = nir_fdiv(nb, src[0], build_length(nb, src[0]));
return;
case GLSLstd450Exp:
val->ssa->def = build_exp(nb, src[0]);
return;
case GLSLstd450Log:
val->ssa->def = build_log(nb, src[0]);
return;
case GLSLstd450FClamp:
val->ssa->def = build_fclamp(nb, src[0], src[1], src[2]);
return;
case GLSLstd450UClamp:
val->ssa->def = nir_umin(nb, nir_umax(nb, src[0], src[1]), src[2]);
return;
case GLSLstd450SClamp:
val->ssa->def = nir_imin(nb, nir_imax(nb, src[0], src[1]), src[2]);
return;
case GLSLstd450Cross: {
unsigned yzx[4] = { 1, 2, 0, 0 };
unsigned zxy[4] = { 2, 0, 1, 0 };
val->ssa->def =
nir_fsub(nb, nir_fmul(nb, nir_swizzle(nb, src[0], yzx, 3, true),
nir_swizzle(nb, src[1], zxy, 3, true)),
nir_fmul(nb, nir_swizzle(nb, src[0], zxy, 3, true),
nir_swizzle(nb, src[1], yzx, 3, true)));
return;
}
case GLSLstd450SmoothStep: {
/* t = clamp((x - edge0) / (edge1 - edge0), 0, 1) */
nir_ssa_def *t =
build_fclamp(nb, nir_fdiv(nb, nir_fsub(nb, src[2], src[0]),
nir_fsub(nb, src[1], src[0])),
nir_imm_float(nb, 0.0), nir_imm_float(nb, 1.0));
/* result = t * t * (3 - 2 * t) */
val->ssa->def =
nir_fmul(nb, t, nir_fmul(nb, t,
nir_fsub(nb, nir_imm_float(nb, 3.0),
nir_fmul(nb, nir_imm_float(nb, 2.0), t))));
return;
}
case GLSLstd450FaceForward:
val->ssa->def =
nir_bcsel(nb, nir_flt(nb, nir_fdot(nb, src[2], src[1]),
nir_imm_float(nb, 0.0)),
src[0], nir_fneg(nb, src[0]));
return;
case GLSLstd450Reflect:
/* I - 2 * dot(N, I) * N */
val->ssa->def =
nir_fsub(nb, src[0], nir_fmul(nb, nir_imm_float(nb, 2.0),
nir_fmul(nb, nir_fdot(nb, src[0], src[1]),
src[1])));
return;
case GLSLstd450Refract: {
nir_ssa_def *I = src[0];
nir_ssa_def *N = src[1];
nir_ssa_def *eta = src[2];
nir_ssa_def *n_dot_i = nir_fdot(nb, N, I);
nir_ssa_def *one = nir_imm_float(nb, 1.0);
nir_ssa_def *zero = nir_imm_float(nb, 0.0);
/* k = 1.0 - eta * eta * (1.0 - dot(N, I) * dot(N, I)) */
nir_ssa_def *k =
nir_fsub(nb, one, nir_fmul(nb, eta, nir_fmul(nb, eta,
nir_fsub(nb, one, nir_fmul(nb, n_dot_i, n_dot_i)))));
nir_ssa_def *result =
nir_fsub(nb, nir_fmul(nb, eta, I),
nir_fmul(nb, nir_fadd(nb, nir_fmul(nb, eta, n_dot_i),
nir_fsqrt(nb, k)), N));
/* XXX: bcsel, or if statement? */
val->ssa->def = nir_bcsel(nb, nir_flt(nb, k, zero), zero, result);
return;
}
case GLSLstd450Sinh:
/* 0.5 * (e^x - e^(-x)) */
val->ssa->def =
nir_fmul(nb, nir_imm_float(nb, 0.5f),
nir_fsub(nb, build_exp(nb, src[0]),
build_exp(nb, nir_fneg(nb, src[0]))));
return;
case GLSLstd450Cosh:
/* 0.5 * (e^x + e^(-x)) */
val->ssa->def =
nir_fmul(nb, nir_imm_float(nb, 0.5f),
nir_fadd(nb, build_exp(nb, src[0]),
build_exp(nb, nir_fneg(nb, src[0]))));
return;
case GLSLstd450Tanh:
/* (0.5 * (e^x - e^(-x))) / (0.5 * (e^x + e^(-x))) */
val->ssa->def =
nir_fdiv(nb, nir_fmul(nb, nir_imm_float(nb, 0.5f),
nir_fsub(nb, build_exp(nb, src[0]),
build_exp(nb, nir_fneg(nb, src[0])))),
nir_fmul(nb, nir_imm_float(nb, 0.5f),
nir_fadd(nb, build_exp(nb, src[0]),
build_exp(nb, nir_fneg(nb, src[0])))));
return;
case GLSLstd450Asinh:
val->ssa->def = nir_fmul(nb, nir_fsign(nb, src[0]),
build_log(nb, nir_fadd(nb, nir_fabs(nb, src[0]),
nir_fsqrt(nb, nir_fadd(nb, nir_fmul(nb, src[0], src[0]),
nir_imm_float(nb, 1.0f))))));
return;
case GLSLstd450Acosh:
val->ssa->def = build_log(nb, nir_fadd(nb, src[0],
nir_fsqrt(nb, nir_fsub(nb, nir_fmul(nb, src[0], src[0]),
nir_imm_float(nb, 1.0f)))));
return;
case GLSLstd450Atanh: {
nir_ssa_def *one = nir_imm_float(nb, 1.0);
val->ssa->def = nir_fmul(nb, nir_imm_float(nb, 0.5f),
build_log(nb, nir_fdiv(nb, nir_fadd(nb, one, src[0]),
nir_fsub(nb, one, src[0]))));
return;
}
case GLSLstd450Asin:
val->ssa->def = build_asin(nb, src[0], 0.086566724, -0.03102955);
return;
case GLSLstd450Acos:
val->ssa->def = nir_fsub(nb, nir_imm_float(nb, M_PI_2f),
build_asin(nb, src[0], 0.08132463, -0.02363318));
return;
case GLSLstd450Atan:
val->ssa->def = build_atan(nb, src[0]);
return;
case GLSLstd450Atan2:
val->ssa->def = build_atan2(nb, src[0], src[1]);
return;
case GLSLstd450Frexp: {
nir_ssa_def *exponent;
val->ssa->def = build_frexp(nb, src[0], &exponent);
nir_store_deref_var(nb, vtn_nir_deref(b, w[6]), exponent, 0xf);
return;
}
case GLSLstd450FrexpStruct: {
assert(glsl_type_is_struct(val->ssa->type));
val->ssa->elems[0]->def = build_frexp(nb, src[0],
&val->ssa->elems[1]->def);
return;
}
default:
val->ssa->def =
nir_build_alu(&b->nb, vtn_nir_alu_op_for_spirv_glsl_opcode(entrypoint),
src[0], src[1], src[2], NULL);
return;
}
}
void
vtn_handle_subgroup(struct vtn_builder *b, SpvOp opcode,
const uint32_t *w, unsigned count)
{
struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
val->ssa = vtn_create_ssa_value(b, val->type->type);
switch (opcode) {
case SpvOpGroupNonUniformElect: {
vtn_fail_if(val->type->type != glsl_bool_type(),
"OpGroupNonUniformElect must return a Bool");
nir_intrinsic_instr *elect =
nir_intrinsic_instr_create(b->nb.shader, nir_intrinsic_elect);
nir_ssa_dest_init_for_type(&elect->instr, &elect->dest,
val->type->type, NULL);
nir_builder_instr_insert(&b->nb, &elect->instr);
val->ssa->def = &elect->dest.ssa;
break;
}
case SpvOpGroupNonUniformBallot: {
vtn_fail_if(val->type->type != glsl_vector_type(GLSL_TYPE_UINT, 4),
"OpGroupNonUniformBallot must return a uvec4");
nir_intrinsic_instr *ballot =
nir_intrinsic_instr_create(b->nb.shader, nir_intrinsic_ballot);
ballot->src[0] = nir_src_for_ssa(vtn_ssa_value(b, w[4])->def);
nir_ssa_dest_init(&ballot->instr, &ballot->dest, 4, 32, NULL);
ballot->num_components = 4;
nir_builder_instr_insert(&b->nb, &ballot->instr);
val->ssa->def = &ballot->dest.ssa;
break;
}
case SpvOpGroupNonUniformInverseBallot: {
/* This one is just a BallotBitfieldExtract with subgroup invocation.
* We could add a NIR intrinsic but it's easier to just lower it on the
* spot.
*/
nir_intrinsic_instr *intrin =
nir_intrinsic_instr_create(b->nb.shader,
nir_intrinsic_ballot_bitfield_extract);
intrin->src[0] = nir_src_for_ssa(vtn_ssa_value(b, w[4])->def);
intrin->src[1] = nir_src_for_ssa(nir_load_subgroup_invocation(&b->nb));
nir_ssa_dest_init_for_type(&intrin->instr, &intrin->dest,
val->type->type, NULL);
nir_builder_instr_insert(&b->nb, &intrin->instr);
val->ssa->def = &intrin->dest.ssa;
break;
}
case SpvOpGroupNonUniformBallotBitExtract:
case SpvOpGroupNonUniformBallotBitCount:
case SpvOpGroupNonUniformBallotFindLSB:
case SpvOpGroupNonUniformBallotFindMSB: {
nir_ssa_def *src0, *src1 = NULL;
nir_intrinsic_op op;
switch (opcode) {
case SpvOpGroupNonUniformBallotBitExtract:
op = nir_intrinsic_ballot_bitfield_extract;
src0 = vtn_ssa_value(b, w[4])->def;
src1 = vtn_ssa_value(b, w[5])->def;
break;
case SpvOpGroupNonUniformBallotBitCount:
switch ((SpvGroupOperation)w[4]) {
case SpvGroupOperationReduce:
op = nir_intrinsic_ballot_bit_count_reduce;
break;
case SpvGroupOperationInclusiveScan:
op = nir_intrinsic_ballot_bit_count_inclusive;
break;
case SpvGroupOperationExclusiveScan:
op = nir_intrinsic_ballot_bit_count_exclusive;
break;
default:
unreachable("Invalid group operation");
}
src0 = vtn_ssa_value(b, w[5])->def;
break;
case SpvOpGroupNonUniformBallotFindLSB:
op = nir_intrinsic_ballot_find_lsb;
src0 = vtn_ssa_value(b, w[4])->def;
break;
case SpvOpGroupNonUniformBallotFindMSB:
op = nir_intrinsic_ballot_find_msb;
src0 = vtn_ssa_value(b, w[4])->def;
break;
default:
unreachable("Unhandled opcode");
}
nir_intrinsic_instr *intrin =
nir_intrinsic_instr_create(b->nb.shader, op);
intrin->src[0] = nir_src_for_ssa(src0);
if (src1)
intrin->src[1] = nir_src_for_ssa(src1);
nir_ssa_dest_init_for_type(&intrin->instr, &intrin->dest,
val->type->type, NULL);
nir_builder_instr_insert(&b->nb, &intrin->instr);
val->ssa->def = &intrin->dest.ssa;
break;
}
case SpvOpGroupNonUniformBroadcastFirst:
vtn_build_subgroup_instr(b, nir_intrinsic_read_first_invocation,
val->ssa, vtn_ssa_value(b, w[4]), NULL, 0, 0);
break;
case SpvOpGroupNonUniformBroadcast:
vtn_build_subgroup_instr(b, nir_intrinsic_read_invocation,
val->ssa, vtn_ssa_value(b, w[4]),
vtn_ssa_value(b, w[5])->def, 0, 0);
break;
case SpvOpGroupNonUniformAll:
case SpvOpGroupNonUniformAny:
case SpvOpGroupNonUniformAllEqual: {
vtn_fail_if(val->type->type != glsl_bool_type(),
"OpGroupNonUniform(All|Any|AllEqual) must return a bool");
nir_intrinsic_op op;
switch (opcode) {
case SpvOpGroupNonUniformAll:
op = nir_intrinsic_vote_all;
break;
case SpvOpGroupNonUniformAny:
op = nir_intrinsic_vote_any;
break;
case SpvOpGroupNonUniformAllEqual: {
switch (glsl_get_base_type(val->type->type)) {
case GLSL_TYPE_FLOAT:
case GLSL_TYPE_DOUBLE:
op = nir_intrinsic_vote_feq;
break;
case GLSL_TYPE_UINT:
case GLSL_TYPE_INT:
case GLSL_TYPE_UINT64:
case GLSL_TYPE_INT64:
case GLSL_TYPE_BOOL:
op = nir_intrinsic_vote_ieq;
break;
default:
unreachable("Unhandled type");
}
break;
}
default:
unreachable("Unhandled opcode");
}
nir_ssa_def *src0 = vtn_ssa_value(b, w[4])->def;
nir_intrinsic_instr *intrin =
nir_intrinsic_instr_create(b->nb.shader, op);
intrin->num_components = src0->num_components;
intrin->src[0] = nir_src_for_ssa(src0);
nir_ssa_dest_init_for_type(&intrin->instr, &intrin->dest,
val->type->type, NULL);
nir_builder_instr_insert(&b->nb, &intrin->instr);
val->ssa->def = &intrin->dest.ssa;
break;
}
case SpvOpGroupNonUniformShuffle:
case SpvOpGroupNonUniformShuffleXor:
case SpvOpGroupNonUniformShuffleUp:
case SpvOpGroupNonUniformShuffleDown: {
nir_intrinsic_op op;
switch (opcode) {
case SpvOpGroupNonUniformShuffle:
op = nir_intrinsic_shuffle;
break;
case SpvOpGroupNonUniformShuffleXor:
op = nir_intrinsic_shuffle_xor;
break;
case SpvOpGroupNonUniformShuffleUp:
op = nir_intrinsic_shuffle_up;
break;
case SpvOpGroupNonUniformShuffleDown:
op = nir_intrinsic_shuffle_down;
break;
default:
unreachable("Invalid opcode");
}
vtn_build_subgroup_instr(b, op, val->ssa, vtn_ssa_value(b, w[4]),
vtn_ssa_value(b, w[5])->def, 0, 0);
break;
}
case SpvOpGroupNonUniformQuadBroadcast:
vtn_build_subgroup_instr(b, nir_intrinsic_quad_broadcast,
val->ssa, vtn_ssa_value(b, w[4]),
vtn_ssa_value(b, w[5])->def, 0, 0);
break;
case SpvOpGroupNonUniformQuadSwap: {
unsigned direction = vtn_constant_uint(b, w[5]);
nir_intrinsic_op op;
switch (direction) {
case 0:
op = nir_intrinsic_quad_swap_horizontal;
break;
case 1:
op = nir_intrinsic_quad_swap_vertical;
break;
case 2:
op = nir_intrinsic_quad_swap_diagonal;
break;
default:
vtn_fail("Invalid constant value in OpGroupNonUniformQuadSwap");
}
vtn_build_subgroup_instr(b, op, val->ssa, vtn_ssa_value(b, w[4]),
NULL, 0, 0);
break;
}
case SpvOpGroupNonUniformIAdd:
case SpvOpGroupNonUniformFAdd:
case SpvOpGroupNonUniformIMul:
case SpvOpGroupNonUniformFMul:
case SpvOpGroupNonUniformSMin:
case SpvOpGroupNonUniformUMin:
case SpvOpGroupNonUniformFMin:
case SpvOpGroupNonUniformSMax:
case SpvOpGroupNonUniformUMax:
case SpvOpGroupNonUniformFMax:
case SpvOpGroupNonUniformBitwiseAnd:
case SpvOpGroupNonUniformBitwiseOr:
case SpvOpGroupNonUniformBitwiseXor:
case SpvOpGroupNonUniformLogicalAnd:
case SpvOpGroupNonUniformLogicalOr:
case SpvOpGroupNonUniformLogicalXor: {
nir_op reduction_op;
switch (opcode) {
case SpvOpGroupNonUniformIAdd:
reduction_op = nir_op_iadd;
break;
case SpvOpGroupNonUniformFAdd:
reduction_op = nir_op_fadd;
break;
case SpvOpGroupNonUniformIMul:
reduction_op = nir_op_imul;
break;
case SpvOpGroupNonUniformFMul:
reduction_op = nir_op_fmul;
break;
case SpvOpGroupNonUniformSMin:
reduction_op = nir_op_imin;
break;
case SpvOpGroupNonUniformUMin:
reduction_op = nir_op_umin;
break;
case SpvOpGroupNonUniformFMin:
reduction_op = nir_op_fmin;
break;
case SpvOpGroupNonUniformSMax:
reduction_op = nir_op_imax;
break;
case SpvOpGroupNonUniformUMax:
reduction_op = nir_op_umax;
break;
case SpvOpGroupNonUniformFMax:
reduction_op = nir_op_fmax;
break;
case SpvOpGroupNonUniformBitwiseAnd:
case SpvOpGroupNonUniformLogicalAnd:
reduction_op = nir_op_iand;
break;
case SpvOpGroupNonUniformBitwiseOr:
case SpvOpGroupNonUniformLogicalOr:
reduction_op = nir_op_ior;
break;
case SpvOpGroupNonUniformBitwiseXor:
case SpvOpGroupNonUniformLogicalXor:
reduction_op = nir_op_ixor;
break;
default:
unreachable("Invalid reduction operation");
}
nir_intrinsic_op op;
unsigned cluster_size = 0;
switch ((SpvGroupOperation)w[4]) {
case SpvGroupOperationReduce:
op = nir_intrinsic_reduce;
break;
case SpvGroupOperationInclusiveScan:
op = nir_intrinsic_inclusive_scan;
break;
case SpvGroupOperationExclusiveScan:
op = nir_intrinsic_exclusive_scan;
break;
case SpvGroupOperationClusteredReduce:
op = nir_intrinsic_reduce;
assert(count == 7);
cluster_size = vtn_constant_uint(b, w[6]);
break;
default:
unreachable("Invalid group operation");
}
vtn_build_subgroup_instr(b, op, val->ssa, vtn_ssa_value(b, w[5]),
NULL, reduction_op, cluster_size);
break;
}
default:
unreachable("Invalid SPIR-V opcode");
}
}
nir_deref_var *
vtn_access_chain_to_deref(struct vtn_builder *b, struct vtn_access_chain *chain)
{
nir_deref_var *deref_var;
if (chain->var->var) {
deref_var = nir_deref_var_create(b, chain->var->var);
} else {
assert(chain->var->members);
/* Create the deref_var manually. It will get filled out later. */
deref_var = rzalloc(b, nir_deref_var);
deref_var->deref.deref_type = nir_deref_type_var;
}
struct vtn_type *deref_type = chain->var->type;
nir_deref *tail = &deref_var->deref;
nir_variable **members = chain->var->members;
for (unsigned i = 0; i < chain->length; i++) {
enum glsl_base_type base_type = glsl_get_base_type(deref_type->type);
switch (base_type) {
case GLSL_TYPE_UINT:
case GLSL_TYPE_INT:
case GLSL_TYPE_FLOAT:
case GLSL_TYPE_DOUBLE:
case GLSL_TYPE_BOOL:
case GLSL_TYPE_ARRAY: {
deref_type = deref_type->array_element;
nir_deref_array *deref_arr = nir_deref_array_create(b);
deref_arr->deref.type = deref_type->type;
if (chain->link[i].mode == vtn_access_mode_literal) {
deref_arr->deref_array_type = nir_deref_array_type_direct;
deref_arr->base_offset = chain->link[i].id;
} else {
assert(chain->link[i].mode == vtn_access_mode_id);
deref_arr->deref_array_type = nir_deref_array_type_indirect;
deref_arr->base_offset = 0;
deref_arr->indirect =
nir_src_for_ssa(vtn_ssa_value(b, chain->link[i].id)->def);
}
tail->child = &deref_arr->deref;
tail = tail->child;
break;
}
case GLSL_TYPE_STRUCT: {
assert(chain->link[i].mode == vtn_access_mode_literal);
unsigned idx = chain->link[i].id;
deref_type = deref_type->members[idx];
if (members) {
/* This is a pre-split structure. */
deref_var->var = members[idx];
rewrite_deref_types(&deref_var->deref, members[idx]->type);
assert(tail->type == deref_type->type);
members = NULL;
} else {
nir_deref_struct *deref_struct = nir_deref_struct_create(b, idx);
deref_struct->deref.type = deref_type->type;
tail->child = &deref_struct->deref;
tail = tail->child;
}
break;
}
default:
unreachable("Invalid type for deref");
}
}
assert(members == NULL);
return deref_var;
}
