static void
convert_instr(nir_builder *bld, nir_alu_instr *alu)
{
nir_ssa_def *numer, *denom, *af, *bf, *a, *b, *q, *r;
nir_op op = alu->op;
bool is_signed;
if ((op != nir_op_idiv) &&
(op != nir_op_udiv) &&
(op != nir_op_umod))
return;
is_signed = (op == nir_op_idiv);
bld->cursor = nir_before_instr(&alu->instr);
numer = nir_ssa_for_alu_src(bld, alu, 0);
denom = nir_ssa_for_alu_src(bld, alu, 1);
if (is_signed) {
af = nir_i2f(bld, numer);
bf = nir_i2f(bld, denom);
af = nir_fabs(bld, af);
bf = nir_fabs(bld, bf);
a = nir_iabs(bld, numer);
b = nir_iabs(bld, denom);
} else {
af = nir_u2f(bld, numer);
bf = nir_u2f(bld, denom);
a = numer;
b = denom;
}
/* get first result: */
bf = nir_frcp(bld, bf);
bf = nir_isub(bld, bf, nir_imm_int(bld, 2)); /* yes, really */
q = nir_fmul(bld, af, bf);
if (is_signed) {
q = nir_f2i(bld, q);
} else {
q = nir_f2u(bld, q);
}
/* get error of first result: */
r = nir_imul(bld, q, b);
r = nir_isub(bld, a, r);
r = nir_u2f(bld, r);
r = nir_fmul(bld, r, bf);
r = nir_f2u(bld, r);
/* add quotients: */
q = nir_iadd(bld, q, r);
/* correction: if modulus >= divisor, add 1 */
r = nir_imul(bld, q, b);
r = nir_isub(bld, a, r);
r = nir_uge(bld, r, b);
r = nir_b2i(bld, r);
q = nir_iadd(bld, q, r);
if (is_signed) {
/* fix the sign: */
r = nir_ixor(bld, numer, denom);
r = nir_ushr(bld, r, nir_imm_int(bld, 31));
r = nir_i2b(bld, r);
b = nir_ineg(bld, q);
q = nir_bcsel(bld, r, b, q);
}
if (op == nir_op_umod) {
/* division result in q */
r = nir_imul(bld, q, b);
q = nir_isub(bld, a, r);
}
assert(alu->dest.dest.is_ssa);
nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(q));
}
void
vtn_handle_alu(struct vtn_builder *b, SpvOp opcode,
const uint32_t *w, unsigned count)
{
struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
const struct glsl_type *type =
vtn_value(b, w[1], vtn_value_type_type)->type->type;
vtn_foreach_decoration(b, val, handle_no_contraction, NULL);
/* Collect the various SSA sources */
const unsigned num_inputs = count - 3;
struct vtn_ssa_value *vtn_src[4] = { NULL, };
for (unsigned i = 0; i < num_inputs; i++)
vtn_src[i] = vtn_ssa_value(b, w[i + 3]);
if (glsl_type_is_matrix(vtn_src[0]->type) ||
(num_inputs >= 2 && glsl_type_is_matrix(vtn_src[1]->type))) {
vtn_handle_matrix_alu(b, opcode, val, vtn_src[0], vtn_src[1]);
b->nb.exact = false;
return;
}
val->ssa = vtn_create_ssa_value(b, type);
nir_ssa_def *src[4] = { NULL, };
for (unsigned i = 0; i < num_inputs; i++) {
assert(glsl_type_is_vector_or_scalar(vtn_src[i]->type));
src[i] = vtn_src[i]->def;
}
switch (opcode) {
case SpvOpAny:
if (src[0]->num_components == 1) {
val->ssa->def = nir_imov(&b->nb, src[0]);
} else {
nir_op op;
switch (src[0]->num_components) {
case 2: op = nir_op_bany_inequal2; break;
case 3: op = nir_op_bany_inequal3; break;
case 4: op = nir_op_bany_inequal4; break;
default: unreachable("invalid number of components");
}
val->ssa->def = nir_build_alu(&b->nb, op, src[0],
nir_imm_int(&b->nb, NIR_FALSE),
NULL, NULL);
}
break;
case SpvOpAll:
if (src[0]->num_components == 1) {
val->ssa->def = nir_imov(&b->nb, src[0]);
} else {
nir_op op;
switch (src[0]->num_components) {
case 2: op = nir_op_ball_iequal2; break;
case 3: op = nir_op_ball_iequal3; break;
case 4: op = nir_op_ball_iequal4; break;
default: unreachable("invalid number of components");
}
val->ssa->def = nir_build_alu(&b->nb, op, src[0],
nir_imm_int(&b->nb, NIR_TRUE),
NULL, NULL);
}
break;
case SpvOpOuterProduct: {
for (unsigned i = 0; i < src[1]->num_components; i++) {
val->ssa->elems[i]->def =
nir_fmul(&b->nb, src[0], nir_channel(&b->nb, src[1], i));
}
break;
}
case SpvOpDot:
val->ssa->def = nir_fdot(&b->nb, src[0], src[1]);
break;
case SpvOpIAddCarry:
assert(glsl_type_is_struct(val->ssa->type));
val->ssa->elems[0]->def = nir_iadd(&b->nb, src[0], src[1]);
val->ssa->elems[1]->def = nir_uadd_carry(&b->nb, src[0], src[1]);
break;
case SpvOpISubBorrow:
assert(glsl_type_is_struct(val->ssa->type));
val->ssa->elems[0]->def = nir_isub(&b->nb, src[0], src[1]);
val->ssa->elems[1]->def = nir_usub_borrow(&b->nb, src[0], src[1]);
break;
case SpvOpUMulExtended:
assert(glsl_type_is_struct(val->ssa->type));
val->ssa->elems[0]->def = nir_imul(&b->nb, src[0], src[1]);
val->ssa->elems[1]->def = nir_umul_high(&b->nb, src[0], src[1]);
break;
case SpvOpSMulExtended:
assert(glsl_type_is_struct(val->ssa->type));
val->ssa->elems[0]->def = nir_imul(&b->nb, src[0], src[1]);
val->ssa->elems[1]->def = nir_imul_high(&b->nb, src[0], src[1]);
break;
case SpvOpFwidth:
val->ssa->def = nir_fadd(&b->nb,
nir_fabs(&b->nb, nir_fddx(&b->nb, src[0])),
nir_fabs(&b->nb, nir_fddy(&b->nb, src[0])));
break;
case SpvOpFwidthFine:
val->ssa->def = nir_fadd(&b->nb,
nir_fabs(&b->nb, nir_fddx_fine(&b->nb, src[0])),
nir_fabs(&b->nb, nir_fddy_fine(&b->nb, src[0])));
break;
case SpvOpFwidthCoarse:
val->ssa->def = nir_fadd(&b->nb,
nir_fabs(&b->nb, nir_fddx_coarse(&b->nb, src[0])),
nir_fabs(&b->nb, nir_fddy_coarse(&b->nb, src[0])));
break;
case SpvOpVectorTimesScalar:
/* The builder will take care of splatting for us. */
val->ssa->def = nir_fmul(&b->nb, src[0], src[1]);
break;
case SpvOpIsNan:
val->ssa->def = nir_fne(&b->nb, src[0], src[0]);
break;
case SpvOpIsInf:
val->ssa->def = nir_feq(&b->nb, nir_fabs(&b->nb, src[0]),
nir_imm_float(&b->nb, INFINITY));
break;
case SpvOpFUnordEqual:
case SpvOpFUnordNotEqual:
case SpvOpFUnordLessThan:
case SpvOpFUnordGreaterThan:
case SpvOpFUnordLessThanEqual:
case SpvOpFUnordGreaterThanEqual: {
bool swap;
nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
nir_op op = vtn_nir_alu_op_for_spirv_opcode(opcode, &swap, src_alu_type, dst_alu_type);
if (swap) {
nir_ssa_def *tmp = src[0];
src[0] = src[1];
src[1] = tmp;
}
val->ssa->def =
nir_ior(&b->nb,
nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL),
nir_ior(&b->nb,
nir_fne(&b->nb, src[0], src[0]),
nir_fne(&b->nb, src[1], src[1])));
break;
}
case SpvOpFOrdEqual:
case SpvOpFOrdNotEqual:
case SpvOpFOrdLessThan:
case SpvOpFOrdGreaterThan:
case SpvOpFOrdLessThanEqual:
case SpvOpFOrdGreaterThanEqual: {
bool swap;
nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
nir_op op = vtn_nir_alu_op_for_spirv_opcode(opcode, &swap, src_alu_type, dst_alu_type);
if (swap) {
nir_ssa_def *tmp = src[0];
src[0] = src[1];
src[1] = tmp;
}
val->ssa->def =
nir_iand(&b->nb,
nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL),
nir_iand(&b->nb,
nir_feq(&b->nb, src[0], src[0]),
nir_feq(&b->nb, src[1], src[1])));
break;
}
default: {
bool swap;
nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
nir_op op = vtn_nir_alu_op_for_spirv_opcode(opcode, &swap, src_alu_type, dst_alu_type);
if (swap) {
nir_ssa_def *tmp = src[0];
src[0] = src[1];
src[1] = tmp;
}
val->ssa->def = nir_build_alu(&b->nb, op, src[0], src[1], src[2], src[3]);
break;
} /* default */
}
b->nb.exact = false;
}
