void
vec4_tes_visitor::nir_emit_intrinsic(nir_intrinsic_instr *instr)
{
const struct brw_tes_prog_data *tes_prog_data =
(const struct brw_tes_prog_data *) prog_data;
switch (instr->intrinsic) {
case nir_intrinsic_load_tess_coord:
/* gl_TessCoord is part of the payload in g1 channels 0-2 and 4-6. */
emit(MOV(get_nir_dest(instr->dest, BRW_REGISTER_TYPE_F),
src_reg(brw_vec8_grf(1, 0))));
break;
case nir_intrinsic_load_tess_level_outer:
if (tes_prog_data->domain == BRW_TESS_DOMAIN_ISOLINE) {
emit(MOV(get_nir_dest(instr->dest, BRW_REGISTER_TYPE_F),
swizzle(src_reg(ATTR, 1, glsl_type::vec4_type),
BRW_SWIZZLE_ZWZW)));
} else {
emit(MOV(get_nir_dest(instr->dest, BRW_REGISTER_TYPE_F),
swizzle(src_reg(ATTR, 1, glsl_type::vec4_type),
BRW_SWIZZLE_WZYX)));
}
break;
case nir_intrinsic_load_tess_level_inner:
if (tes_prog_data->domain == BRW_TESS_DOMAIN_QUAD) {
emit(MOV(get_nir_dest(instr->dest, BRW_REGISTER_TYPE_F),
swizzle(src_reg(ATTR, 0, glsl_type::vec4_type),
BRW_SWIZZLE_WZYX)));
} else {
emit(MOV(get_nir_dest(instr->dest, BRW_REGISTER_TYPE_F),
src_reg(ATTR, 1, glsl_type::float_type)));
}
break;
case nir_intrinsic_load_primitive_id:
emit(TES_OPCODE_GET_PRIMITIVE_ID,
get_nir_dest(instr->dest, BRW_REGISTER_TYPE_UD));
break;
case nir_intrinsic_load_input:
case nir_intrinsic_load_per_vertex_input: {
src_reg indirect_offset = get_indirect_offset(instr);
unsigned imm_offset = instr->const_index[0];
src_reg header = input_read_header;
bool is_64bit = nir_dest_bit_size(instr->dest) == 64;
unsigned first_component = nir_intrinsic_component(instr);
if (is_64bit)
first_component /= 2;
if (indirect_offset.file != BAD_FILE) {
header = src_reg(this, glsl_type::uvec4_type);
emit(TES_OPCODE_ADD_INDIRECT_URB_OFFSET, dst_reg(header),
input_read_header, indirect_offset);
} else {
/* Arbitrarily only push up to 24 vec4 slots worth of data,
* which is 12 registers (since each holds 2 vec4 slots).
*/
const unsigned max_push_slots = 24;
if (imm_offset < max_push_slots) {
const glsl_type *src_glsl_type =
is_64bit ? glsl_type::dvec4_type : glsl_type::ivec4_type;
src_reg src = src_reg(ATTR, imm_offset, src_glsl_type);
src.swizzle = BRW_SWZ_COMP_INPUT(first_component);
const brw_reg_type dst_reg_type =
is_64bit ? BRW_REGISTER_TYPE_DF : BRW_REGISTER_TYPE_D;
emit(MOV(get_nir_dest(instr->dest, dst_reg_type), src));
prog_data->urb_read_length =
MAX2(prog_data->urb_read_length,
DIV_ROUND_UP(imm_offset + (is_64bit ? 2 : 1), 2));
break;
}
}
if (!is_64bit) {
dst_reg temp(this, glsl_type::ivec4_type);
vec4_instruction *read =
emit(VEC4_OPCODE_URB_READ, temp, src_reg(header));
read->offset = imm_offset;
read->urb_write_flags = BRW_URB_WRITE_PER_SLOT_OFFSET;
src_reg src = src_reg(temp);
src.swizzle = BRW_SWZ_COMP_INPUT(first_component);
/* Copy to target. We might end up with some funky writemasks landing
* in here, but we really don't want them in the above pseudo-ops.
*/
dst_reg dst = get_nir_dest(instr->dest, BRW_REGISTER_TYPE_D);
dst.writemask = brw_writemask_for_size(instr->num_components);
emit(MOV(dst, src));
} else {
/* For 64-bit we need to load twice as many 32-bit components, and for
* dvec3/4 we need to emit 2 URB Read messages
*/
dst_reg temp(this, glsl_type::dvec4_type);
dst_reg temp_d = retype(temp, BRW_REGISTER_TYPE_D);
vec4_instruction *read =
emit(VEC4_OPCODE_URB_READ, temp_d, src_reg(header));
read->offset = imm_offset;
read->urb_write_flags = BRW_URB_WRITE_PER_SLOT_OFFSET;
if (instr->num_components > 2) {
read = emit(VEC4_OPCODE_URB_READ, byte_offset(temp_d, REG_SIZE),
src_reg(header));
read->offset = imm_offset + 1;
read->urb_write_flags = BRW_URB_WRITE_PER_SLOT_OFFSET;
}
src_reg temp_as_src = src_reg(temp);
temp_as_src.swizzle = BRW_SWZ_COMP_INPUT(first_component);
dst_reg shuffled(this, glsl_type::dvec4_type);
shuffle_64bit_data(shuffled, temp_as_src, false);
dst_reg dst = get_nir_dest(instr->dest, BRW_REGISTER_TYPE_DF);
dst.writemask = brw_writemask_for_size(instr->num_components);
emit(MOV(dst, src_reg(shuffled)));
}
break;
}
default:
vec4_visitor::nir_emit_intrinsic(instr);
}
}
void
vec4_gs_visitor::nir_emit_intrinsic(nir_intrinsic_instr *instr)
{
dst_reg dest;
src_reg src;
switch (instr->intrinsic) {
case nir_intrinsic_emit_vertex: {
int stream_id = instr->const_index[0];
gs_emit_vertex(stream_id);
break;
}
case nir_intrinsic_end_primitive:
gs_end_primitive();
break;
case nir_intrinsic_load_invocation_id: {
src_reg invocation_id =
src_reg(nir_system_values[SYSTEM_VALUE_INVOCATION_ID]);
assert(invocation_id.file != BAD_FILE);
dest = get_nir_dest(instr->dest, invocation_id.type);
emit(MOV(dest, invocation_id));
break;
}
default:
vec4_visitor::nir_emit_intrinsic(instr);
}
}
void
vec4_gs_visitor::nir_emit_intrinsic(nir_intrinsic_instr *instr)
{
dst_reg dest;
src_reg src;
switch (instr->intrinsic) {
case nir_intrinsic_load_per_vertex_input: {
/* The EmitNoIndirectInput flag guarantees our vertex index will
* be constant. We should handle indirects someday.
*/
nir_const_value *vertex = nir_src_as_const_value(instr->src[0]);
nir_const_value *offset = nir_src_as_const_value(instr->src[1]);
/* Make up a type...we have no way of knowing... */
const glsl_type *const type = glsl_type::ivec(instr->num_components);
src = src_reg(ATTR, BRW_VARYING_SLOT_COUNT * vertex->u32[0] +
instr->const_index[0] + offset->u32[0],
type);
/* gl_PointSize is passed in the .w component of the VUE header */
if (instr->const_index[0] == VARYING_SLOT_PSIZ)
src.swizzle = BRW_SWIZZLE_WWWW;
dest = get_nir_dest(instr->dest, src.type);
dest.writemask = brw_writemask_for_size(instr->num_components);
emit(MOV(dest, src));
break;
}
case nir_intrinsic_load_input:
unreachable("nir_lower_io should have produced per_vertex intrinsics");
case nir_intrinsic_emit_vertex_with_counter: {
this->vertex_count =
retype(get_nir_src(instr->src[0], 1), BRW_REGISTER_TYPE_UD);
int stream_id = instr->const_index[0];
gs_emit_vertex(stream_id);
break;
}
case nir_intrinsic_end_primitive_with_counter:
this->vertex_count =
retype(get_nir_src(instr->src[0], 1), BRW_REGISTER_TYPE_UD);
gs_end_primitive();
break;
case nir_intrinsic_set_vertex_count:
this->vertex_count =
retype(get_nir_src(instr->src[0], 1), BRW_REGISTER_TYPE_UD);
break;
case nir_intrinsic_load_primitive_id:
assert(gs_prog_data->include_primitive_id);
dest = get_nir_dest(instr->dest, BRW_REGISTER_TYPE_D);
emit(MOV(dest, retype(brw_vec4_grf(1, 0), BRW_REGISTER_TYPE_D)));
break;
case nir_intrinsic_load_invocation_id: {
src_reg invocation_id =
src_reg(nir_system_values[SYSTEM_VALUE_INVOCATION_ID]);
assert(invocation_id.file != BAD_FILE);
dest = get_nir_dest(instr->dest, invocation_id.type);
emit(MOV(dest, invocation_id));
break;
}
default:
vec4_visitor::nir_emit_intrinsic(instr);
}
}
void
vec4_tcs_visitor::nir_emit_intrinsic(nir_intrinsic_instr *instr)
{
switch (instr->intrinsic) {
case nir_intrinsic_load_invocation_id:
emit(MOV(get_nir_dest(instr->dest, BRW_REGISTER_TYPE_UD),
invocation_id));
break;
case nir_intrinsic_load_primitive_id:
emit(TCS_OPCODE_GET_PRIMITIVE_ID,
get_nir_dest(instr->dest, BRW_REGISTER_TYPE_UD));
break;
case nir_intrinsic_load_patch_vertices_in:
emit(MOV(get_nir_dest(instr->dest, BRW_REGISTER_TYPE_D),
brw_imm_d(key->input_vertices)));
break;
case nir_intrinsic_load_per_vertex_input: {
src_reg indirect_offset = get_indirect_offset(instr);
unsigned imm_offset = instr->const_index[0];
nir_const_value *vertex_const = nir_src_as_const_value(instr->src[0]);
src_reg vertex_index =
vertex_const ? src_reg(brw_imm_ud(vertex_const->u32[0]))
: get_nir_src(instr->src[0], BRW_REGISTER_TYPE_UD, 1);
dst_reg dst = get_nir_dest(instr->dest, BRW_REGISTER_TYPE_D);
dst.writemask = brw_writemask_for_size(instr->num_components);
emit_input_urb_read(dst, vertex_index, imm_offset,
nir_intrinsic_component(instr), indirect_offset);
break;
}
case nir_intrinsic_load_input:
unreachable("nir_lower_io should use load_per_vertex_input intrinsics");
break;
case nir_intrinsic_load_output:
case nir_intrinsic_load_per_vertex_output: {
src_reg indirect_offset = get_indirect_offset(instr);
unsigned imm_offset = instr->const_index[0];
dst_reg dst = get_nir_dest(instr->dest, BRW_REGISTER_TYPE_D);
dst.writemask = brw_writemask_for_size(instr->num_components);
if (imm_offset == 0 && indirect_offset.file == BAD_FILE) {
dst.type = BRW_REGISTER_TYPE_F;
/* This is a read of gl_TessLevelInner[], which lives in the
* Patch URB header. The layout depends on the domain.
*/
switch (key->tes_primitive_mode) {
case GL_QUADS: {
/* DWords 3-2 (reversed); use offset 0 and WZYX swizzle. */
dst_reg tmp(this, glsl_type::vec4_type);
emit_output_urb_read(tmp, 0, 0, src_reg());
emit(MOV(writemask(dst, WRITEMASK_XY),
swizzle(src_reg(tmp), BRW_SWIZZLE_WZYX)));
break;
}
case GL_TRIANGLES:
/* DWord 4; use offset 1 but normal swizzle/writemask. */
emit_output_urb_read(writemask(dst, WRITEMASK_X), 1, 0,
src_reg());
break;
case GL_ISOLINES:
/* All channels are undefined. */
return;
default:
unreachable("Bogus tessellation domain");
}
} else if (imm_offset == 1 && indirect_offset.file == BAD_FILE) {
dst.type = BRW_REGISTER_TYPE_F;
unsigned swiz = BRW_SWIZZLE_WZYX;
/* This is a read of gl_TessLevelOuter[], which lives in the
* high 4 DWords of the Patch URB header, in reverse order.
*/
switch (key->tes_primitive_mode) {
case GL_QUADS:
dst.writemask = WRITEMASK_XYZW;
break;
case GL_TRIANGLES:
dst.writemask = WRITEMASK_XYZ;
break;
case GL_ISOLINES:
/* Isolines are not reversed; swizzle .zw -> .xy */
swiz = BRW_SWIZZLE_ZWZW;
dst.writemask = WRITEMASK_XY;
return;
default:
unreachable("Bogus tessellation domain");
}
dst_reg tmp(this, glsl_type::vec4_type);
emit_output_urb_read(tmp, 1, 0, src_reg());
emit(MOV(dst, swizzle(src_reg(tmp), swiz)));
} else {
emit_output_urb_read(dst, imm_offset, nir_intrinsic_component(instr),
indirect_offset);
}
break;
}
case nir_intrinsic_store_output:
case nir_intrinsic_store_per_vertex_output: {
src_reg value = get_nir_src(instr->src[0]);
unsigned mask = instr->const_index[1];
unsigned swiz = BRW_SWIZZLE_XYZW;
src_reg indirect_offset = get_indirect_offset(instr);
unsigned imm_offset = instr->const_index[0];
/* The passthrough shader writes the whole patch header as two vec4s;
* skip all the gl_TessLevelInner/Outer swizzling.
*/
if (indirect_offset.file == BAD_FILE && !is_passthrough_shader) {
if (imm_offset == 0) {
value.type = BRW_REGISTER_TYPE_F;
mask &=
(1 << tesslevel_inner_components(key->tes_primitive_mode)) - 1;
/* This is a write to gl_TessLevelInner[], which lives in the
* Patch URB header. The layout depends on the domain.
*/
switch (key->tes_primitive_mode) {
case GL_QUADS:
/* gl_TessLevelInner[].xy lives at DWords 3-2 (reversed).
* We use an XXYX swizzle to reverse put .xy in the .wz
* channels, and use a .zw writemask.
*/
swiz = BRW_SWIZZLE4(0, 0, 1, 0);
mask = writemask_for_backwards_vector(mask);
break;
case GL_TRIANGLES:
/* gl_TessLevelInner[].x lives at DWord 4, so we set the
* writemask to X and bump the URB offset by 1.
*/
imm_offset = 1;
break;
case GL_ISOLINES:
/* Skip; gl_TessLevelInner[] doesn't exist for isolines. */
return;
default:
unreachable("Bogus tessellation domain");
}
} else if (imm_offset == 1) {
value.type = BRW_REGISTER_TYPE_F;
mask &=
(1 << tesslevel_outer_components(key->tes_primitive_mode)) - 1;
/* This is a write to gl_TessLevelOuter[] which lives in the
* Patch URB Header at DWords 4-7. However, it's reversed, so
* instead of .xyzw we have .wzyx.
*/
if (key->tes_primitive_mode == GL_ISOLINES) {
/* Isolines .xy should be stored in .zw, in order. */
swiz = BRW_SWIZZLE4(0, 0, 0, 1);
mask <<= 2;
} else {
/* Other domains are reversed; store .wzyx instead of .xyzw. */
swiz = BRW_SWIZZLE_WZYX;
mask = writemask_for_backwards_vector(mask);
}
}
}
unsigned first_component = nir_intrinsic_component(instr);
if (first_component) {
assert(swiz == BRW_SWIZZLE_XYZW);
swiz = BRW_SWZ_COMP_OUTPUT(first_component);
mask = mask << first_component;
}
emit_urb_write(swizzle(value, swiz), mask,
imm_offset, indirect_offset);
break;
}
case nir_intrinsic_barrier: {
dst_reg header = dst_reg(this, glsl_type::uvec4_type);
emit(TCS_OPCODE_CREATE_BARRIER_HEADER, header);
emit(SHADER_OPCODE_BARRIER, dst_null_ud(), src_reg(header));
break;
}
default:
vec4_visitor::nir_emit_intrinsic(instr);
}
}