nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
if ((mode == nir_var_shader_in && is_input(intrin)) ||
(mode == nir_var_shader_out && is_output(intrin))) {
nir_src *offset = nir_get_io_offset_src(intrin);
nir_const_value *const_offset = nir_src_as_const_value(*offset);
if (const_offset) {
intrin->const_index[0] += const_offset->u32[0];
b->cursor = nir_before_instr(&intrin->instr);
nir_instr_rewrite_src(&intrin->instr, offset,
nir_src_for_ssa(nir_imm_int(b, 0)));
}
}
}
/* turns 'fddy(p)' into 'fddy(fmul(p, transform.x))' */
static void
lower_fddy(lower_wpos_ytransform_state *state, nir_alu_instr *fddy)
{
nir_builder *b = &state->b;
nir_ssa_def *p, *pt, *trans;
b->cursor = nir_before_instr(&fddy->instr);
p = nir_ssa_for_alu_src(b, fddy, 0);
trans = get_transform(state);
pt = nir_fmul(b, p, nir_channel(b, trans, 0));
nir_instr_rewrite_src(&fddy->instr,
&fddy->src[0].src,
nir_src_for_ssa(pt));
for (unsigned i = 0; i < 4; i++)
fddy->src[0].swizzle[i] = MIN2(i, pt->num_components - 1);
}
static bool
lower_indirect_block(nir_block *block, nir_builder *b,
nir_variable_mode modes)
{
bool progress = false;
nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
if (intrin->intrinsic != nir_intrinsic_load_var &&
intrin->intrinsic != nir_intrinsic_store_var)
continue;
if (!deref_has_indirect(intrin->variables[0]))
continue;
/* Only lower variables whose mode is in the mask */
if (!(modes & intrin->variables[0]->var->data.mode))
continue;
b->cursor = nir_before_instr(&intrin->instr);
if (intrin->intrinsic == nir_intrinsic_load_var) {
nir_ssa_def *result;
emit_load_store(b, intrin, intrin->variables[0],
&intrin->variables[0]->deref, &result, NULL);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(result));
} else {
assert(intrin->src[0].is_ssa);
emit_load_store(b, intrin, intrin->variables[0],
&intrin->variables[0]->deref, NULL, intrin->src[0].ssa);
}
nir_instr_remove(&intrin->instr);
progress = true;
}
return progress;
}
static nir_ssa_def *
get_texture_size(nir_builder *b, nir_tex_instr *tex)
{
b->cursor = nir_before_instr(&tex->instr);
/* RECT textures should not be array: */
assert(!tex->is_array);
nir_tex_instr *txs;
txs = nir_tex_instr_create(b->shader, 1);
txs->op = nir_texop_txs;
txs->sampler_dim = GLSL_SAMPLER_DIM_RECT;
txs->sampler_index = tex->sampler_index;
/* only single src, the lod: */
txs->src[0].src = nir_src_for_ssa(nir_imm_int(b, 0));
txs->src[0].src_type = nir_tex_src_lod;
nir_ssa_dest_init(&txs->instr, &txs->dest, 2, NULL);
nir_builder_instr_insert(b, &txs->instr);
return nir_i2f(b, &txs->dest.ssa);
}
static void
project_src(nir_builder *b, nir_tex_instr *tex)
{
/* Find the projector in the srcs list, if present. */
int proj_index = nir_tex_instr_src_index(tex, nir_tex_src_projector);
if (proj_index < 0)
return;
b->cursor = nir_before_instr(&tex->instr);
nir_ssa_def *inv_proj =
nir_frcp(b, nir_ssa_for_src(b, tex->src[proj_index].src, 1));
/* Walk through the sources projecting the arguments. */
for (unsigned i = 0; i < tex->num_srcs; i++) {
switch (tex->src[i].src_type) {
case nir_tex_src_coord:
case nir_tex_src_comparator:
break;
default:
continue;
}
nir_ssa_def *unprojected =
nir_ssa_for_src(b, tex->src[i].src, nir_tex_instr_src_size(tex, i));
nir_ssa_def *projected = nir_fmul(b, unprojected, inv_proj);
/* Array indices don't get projected, so make an new vector with the
* coordinate's array index untouched.
*/
if (tex->is_array && tex->src[i].src_type == nir_tex_src_coord) {
switch (tex->coord_components) {
case 4:
projected = nir_vec4(b,
nir_channel(b, projected, 0),
nir_channel(b, projected, 1),
nir_channel(b, projected, 2),
nir_channel(b, unprojected, 3));
break;
case 3:
projected = nir_vec3(b,
nir_channel(b, projected, 0),
nir_channel(b, projected, 1),
nir_channel(b, unprojected, 2));
break;
case 2:
projected = nir_vec2(b,
nir_channel(b, projected, 0),
nir_channel(b, unprojected, 1));
break;
default:
unreachable("bad texture coord count for array");
break;
}
}
nir_instr_rewrite_src(&tex->instr,
&tex->src[i].src,
nir_src_for_ssa(projected));
}
nir_tex_instr_remove_src(tex, proj_index);
}
/* see emit_wpos_adjustment() in st_mesa_to_tgsi.c */
static void
emit_wpos_adjustment(lower_wpos_ytransform_state *state,
nir_intrinsic_instr *intr,
bool invert, float adjX, float adjY[2])
{
nir_builder *b = &state->b;
nir_variable *fragcoord = intr->variables[0]->var;
nir_ssa_def *wpostrans, *wpos_temp, *wpos_temp_y, *wpos_input;
assert(intr->dest.is_ssa);
b->cursor = nir_before_instr(&intr->instr);
wpostrans = get_transform(state);
wpos_input = nir_load_var(b, fragcoord);
/* First, apply the coordinate shift: */
if (adjX || adjY[0] || adjY[1]) {
if (adjY[0] != adjY[1]) {
/* Adjust the y coordinate by adjY[1] or adjY[0] respectively
* depending on whether inversion is actually going to be applied
* or not, which is determined by testing against the inversion
* state variable used below, which will be either +1 or -1.
*/
nir_ssa_def *adj_temp;
adj_temp = nir_cmp(b,
nir_channel(b, wpostrans, invert ? 2 : 0),
nir_imm_vec4(b, adjX, adjY[0], 0.0f, 0.0f),
nir_imm_vec4(b, adjX, adjY[1], 0.0f, 0.0f));
wpos_temp = nir_fadd(b, wpos_input, adj_temp);
} else {
wpos_temp = nir_fadd(b,
wpos_input,
nir_imm_vec4(b, adjX, adjY[0], 0.0f, 0.0f));
}
wpos_input = wpos_temp;
} else {
/* MOV wpos_temp, input[wpos]
*/
wpos_temp = wpos_input;
}
/* Now the conditional y flip: STATE_FB_WPOS_Y_TRANSFORM.xy/zw will be
* inversion/identity, or the other way around if we're drawing to an FBO.
*/
if (invert) {
/* wpos_temp.y = wpos_input * wpostrans.xxxx + wpostrans.yyyy */
wpos_temp_y = nir_fadd(b, nir_fmul(b, nir_channel(b, wpos_temp, 1),
nir_channel(b, wpostrans, 0)),
nir_channel(b, wpostrans, 1));
} else {
/* wpos_temp.y = wpos_input * wpostrans.zzzz + wpostrans.wwww */
wpos_temp_y = nir_fadd(b, nir_fmul(b, nir_channel(b, wpos_temp, 1),
nir_channel(b, wpostrans, 2)),
nir_channel(b, wpostrans, 3));
}
wpos_temp = nir_vec4(b,
nir_channel(b, wpos_temp, 0),
wpos_temp_y,
nir_channel(b, wpos_temp, 2),
nir_channel(b, wpos_temp, 3));
nir_ssa_def_rewrite_uses(&intr->dest.ssa, nir_src_for_ssa(wpos_temp));
}
static void
convert_instr(nir_builder *bld, nir_alu_instr *alu)
{
nir_ssa_def *numer, *denom, *af, *bf, *a, *b, *q, *r;
nir_op op = alu->op;
bool is_signed;
if ((op != nir_op_idiv) &&
(op != nir_op_udiv) &&
(op != nir_op_umod))
return;
is_signed = (op == nir_op_idiv);
bld->cursor = nir_before_instr(&alu->instr);
numer = nir_ssa_for_alu_src(bld, alu, 0);
denom = nir_ssa_for_alu_src(bld, alu, 1);
if (is_signed) {
af = nir_i2f(bld, numer);
bf = nir_i2f(bld, denom);
af = nir_fabs(bld, af);
bf = nir_fabs(bld, bf);
a = nir_iabs(bld, numer);
b = nir_iabs(bld, denom);
} else {
af = nir_u2f(bld, numer);
bf = nir_u2f(bld, denom);
a = numer;
b = denom;
}
/* get first result: */
bf = nir_frcp(bld, bf);
bf = nir_isub(bld, bf, nir_imm_int(bld, 2)); /* yes, really */
q = nir_fmul(bld, af, bf);
if (is_signed) {
q = nir_f2i(bld, q);
} else {
q = nir_f2u(bld, q);
}
/* get error of first result: */
r = nir_imul(bld, q, b);
r = nir_isub(bld, a, r);
r = nir_u2f(bld, r);
r = nir_fmul(bld, r, bf);
r = nir_f2u(bld, r);
/* add quotients: */
q = nir_iadd(bld, q, r);
/* correction: if modulus >= divisor, add 1 */
r = nir_imul(bld, q, b);
r = nir_isub(bld, a, r);
r = nir_uge(bld, r, b);
r = nir_b2i(bld, r);
q = nir_iadd(bld, q, r);
if (is_signed) {
/* fix the sign: */
r = nir_ixor(bld, numer, denom);
r = nir_ushr(bld, r, nir_imm_int(bld, 31));
r = nir_i2b(bld, r);
b = nir_ineg(bld, q);
q = nir_bcsel(bld, r, b, q);
}
if (op == nir_op_umod) {
/* division result in q */
r = nir_imul(bld, q, b);
q = nir_isub(bld, a, r);
}
assert(alu->dest.dest.is_ssa);
nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(q));
}
static bool
lower_locals_to_regs_block(nir_block *block,
struct locals_to_regs_state *state)
{
nir_builder *b = &state->builder;
nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_load_deref: {
nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
if (deref->mode != nir_var_function_temp)
continue;
b->cursor = nir_before_instr(&intrin->instr);
nir_alu_instr *mov = nir_alu_instr_create(b->shader, nir_op_imov);
mov->src[0].src = get_deref_reg_src(deref, state);
mov->dest.write_mask = (1 << intrin->num_components) - 1;
if (intrin->dest.is_ssa) {
nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
intrin->num_components,
intrin->dest.ssa.bit_size, NULL);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
nir_src_for_ssa(&mov->dest.dest.ssa));
} else {
nir_dest_copy(&mov->dest.dest, &intrin->dest, &mov->instr);
}
nir_builder_instr_insert(b, &mov->instr);
nir_instr_remove(&intrin->instr);
state->progress = true;
break;
}
case nir_intrinsic_store_deref: {
nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
if (deref->mode != nir_var_function_temp)
continue;
b->cursor = nir_before_instr(&intrin->instr);
nir_src reg_src = get_deref_reg_src(deref, state);
nir_alu_instr *mov = nir_alu_instr_create(b->shader, nir_op_imov);
nir_src_copy(&mov->src[0].src, &intrin->src[1], mov);
mov->dest.write_mask = nir_intrinsic_write_mask(intrin);
mov->dest.dest.is_ssa = false;
mov->dest.dest.reg.reg = reg_src.reg.reg;
mov->dest.dest.reg.base_offset = reg_src.reg.base_offset;
mov->dest.dest.reg.indirect = reg_src.reg.indirect;
nir_builder_instr_insert(b, &mov->instr);
nir_instr_remove(&intrin->instr);
state->progress = true;
break;
}
case nir_intrinsic_copy_deref:
unreachable("There should be no copies whatsoever at this point");
break;
default:
continue;
}
}
return true;
}
static void
vc4_nir_lower_txf_ms_instr(struct vc4_compile *c, nir_builder *b,
nir_tex_instr *txf_ms)
{
if (txf_ms->op != nir_texop_txf_ms)
return;
b->cursor = nir_before_instr(&txf_ms->instr);
nir_tex_instr *txf = nir_tex_instr_create(c->s, 1);
txf->op = nir_texop_txf;
txf->sampler = txf_ms->sampler;
txf->sampler_index = txf_ms->sampler_index;
txf->coord_components = txf_ms->coord_components;
txf->is_shadow = txf_ms->is_shadow;
txf->is_new_style_shadow = txf_ms->is_new_style_shadow;
nir_ssa_def *coord = NULL, *sample_index = NULL;
for (int i = 0; i < txf_ms->num_srcs; i++) {
assert(txf_ms->src[i].src.is_ssa);
switch (txf_ms->src[i].src_type) {
case nir_tex_src_coord:
coord = txf_ms->src[i].src.ssa;
break;
case nir_tex_src_ms_index:
sample_index = txf_ms->src[i].src.ssa;
break;
default:
unreachable("Unknown txf_ms src\n");
}
}
assert(coord);
assert(sample_index);
nir_ssa_def *x = nir_channel(b, coord, 0);
nir_ssa_def *y = nir_channel(b, coord, 1);
uint32_t tile_w = 32;
uint32_t tile_h = 32;
uint32_t tile_w_shift = 5;
uint32_t tile_h_shift = 5;
uint32_t tile_size = (tile_h * tile_w *
VC4_MAX_SAMPLES * sizeof(uint32_t));
unsigned unit = txf_ms->sampler_index;
uint32_t w = align(c->key->tex[unit].msaa_width, tile_w);
uint32_t w_tiles = w / tile_w;
nir_ssa_def *x_tile = nir_ushr(b, x, nir_imm_int(b, tile_w_shift));
nir_ssa_def *y_tile = nir_ushr(b, y, nir_imm_int(b, tile_h_shift));
nir_ssa_def *tile_addr = nir_iadd(b,
nir_imul(b, x_tile,
nir_imm_int(b, tile_size)),
nir_imul(b, y_tile,
nir_imm_int(b, (w_tiles *
tile_size))));
nir_ssa_def *x_subspan = nir_iand(b, x,
nir_imm_int(b, (tile_w - 1) & ~1));
nir_ssa_def *y_subspan = nir_iand(b, y,
nir_imm_int(b, (tile_h - 1) & ~1));
nir_ssa_def *subspan_addr = nir_iadd(b,
nir_imul(b, x_subspan,
nir_imm_int(b, 2 * VC4_MAX_SAMPLES * sizeof(uint32_t))),
nir_imul(b, y_subspan,
nir_imm_int(b,
tile_w *
VC4_MAX_SAMPLES *
sizeof(uint32_t))));
nir_ssa_def *pixel_addr = nir_ior(b,
nir_iand(b,
nir_ishl(b, x,
nir_imm_int(b, 2)),
nir_imm_int(b, (1 << 2))),
nir_iand(b,
nir_ishl(b, y,
nir_imm_int(b, 3)),
nir_imm_int(b, (1 << 3))));
nir_ssa_def *sample_addr = nir_ishl(b, sample_index, nir_imm_int(b, 4));
nir_ssa_def *addr = nir_iadd(b,
nir_ior(b, sample_addr, pixel_addr),
nir_iadd(b, subspan_addr, tile_addr));
txf->src[0].src_type = nir_tex_src_coord;
txf->src[0].src = nir_src_for_ssa(nir_vec2(b, addr, nir_imm_int(b, 0)));
nir_ssa_dest_init(&txf->instr, &txf->dest, 4, NULL);
nir_builder_instr_insert(b, &txf->instr);
nir_ssa_def_rewrite_uses(&txf_ms->dest.ssa,
nir_src_for_ssa(&txf->dest.ssa));
nir_instr_remove(&txf_ms->instr);
}